1/1: import matplotlib
 1/2: plt.plot()
 1/3: import matplotlib.pyplot as plt
 1/4: plt.plot()
 1/5: plt.show()
 2/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram -nfft 32768 -hp 50 -lp 8000 -cpu 32
 3/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram -nfft 32768 -hp 50 -lp 8000 -cpu 16
 4/1: import librosa
 5/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram_GUA_AB_20210211_20210316 -nfft 32768 -hp 50 -lp 8000 -cpu 40
 5/2: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram_GUA_AB_20210211_20210316 -nfft 32768 -hp 50 -lp 8000 -cpu 32
 6/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram_GUA_AB_20210211_20210316 -nfft 32768 -hp 50 -lp 8000 -cpu 32
 8/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram_GUA_AB_20210211_20210316 -nfft 32768 -hp 50 -lp 8000 -cpu 32
 9/1: %run yolo-dyni/get_spectrogram.py /nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUA_AB_20210211_20210316/ -o ./Spectrogram_GUA_AB_20210211_20210316 -nfft 32768 -hp 50 -lp 8000 -cpu 32
10/1: WAV = [0,1,2,3]
10/2: wav = [1,2,3,4]
10/3: [WAV, wav]
10/4: np.concate([WAV, wav])
10/5: import numpy as np
10/6: np.concate([WAV, wav])
10/7: np.concatenate([WAV, wav])
11/1: import pytorch
11/2: import torch
12/1: ls
12/2: ls
12/3: df = pd.DataFrame(columns = ['file','idx','espece','x','y','w','h','conf'])
12/4: files = glob.glob('yolov5/runs/detect/exp_Fjord3D_051122/labels/*.txt')
12/5: import pandas as pd
13/1: import pandas as pd
13/2: import glob
13/3: df = pd.DataFrame(columns = ['file','idx','espece','x','y','w','h','conf'])
13/4: files = glob.glob('yolov5/runs/detect/exp7/labels/*.txt')
13/5: files = glob.glob('yolov5/runs/detect/exp_Fjord3D_051122/labels/*.txt')
13/6: files
13/7:
for f in files:
    tab = pd.read_csv(f, sep=' ', names = ['espece','x','y','w','h','conf'])
    tab['file'] = f
    tab['idx'] = int(f.split('-')[-1][:-4])
    df = pd.concat([df, tab])
13/8: df
13/9: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
13/10: df
13/11: df = df.sort_values('file')
13/12: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
13/13: import matplotlib.pyplot as plt
13/14: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
13/15: import numpy as np
13/16: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
13/17: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
13/18: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.title("Confience de détection pour FJORD 05_11_22");plt.show()
13/19: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.title("Confience de détection pour FJORD 05_11_22");plt.figsave("hist_confience_FJORD_05112022.pdf")
13/20: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.title("Confience de détection pour FJORD 05_11_22");plt.savefig("hist_confience_FJORD_05112022.pdf")
13/21: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.title("Confience de détection pour FJORD 05_11_22");plt.savefig("hist_confience_FJORD_05112022.pdf")
13/22:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
13/23: res
13/24: plt.close()
13/25: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.title("Confience de détection pour FJORD 05_11_22");plt.savefig("hist_confience_FJORD_05112022.pdf")
13/26: df['pos'] = df['conf'] >= 0.7
13/27:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
13/28: res
13/29: res.sort_values("pos")
13/30:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
13/31: res.sort_values("pos")
13/32:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
13/33: res
13/34: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection GUA_AB_2021-02-11"); plt.show()
13/35: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_05_11_2022"); plt.savefig("detection_Fjord_05_11_2022_heure.pdf")
13/36:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
13/37: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_05_11_2022"); plt.savefig("detection_Fjord_05_11_2022_minute.pdf")
13/38: %history -f save-ipython/export_results_V2.txt
14/1:
import pandas as pd
import glob
df = pd.DataFrame(columns = ['file','idx','espece','x','y','w','h','conf'])
14/2: import matplotlib.pyplot as plt
14/3: files = glob.glob('yolov5/runs/detect/exp_Fjord3D_041122/labels/*.txt')
14/4:
for f in files:
    tab = pd.read_csv(f, sep=' ', names = ['espece','x','y','w','h','conf'])
    tab['file'] = f
    tab['idx'] = int(f.split('-')[-1][:-4])
    df = pd.concat([df, tab])
14/5: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
14/6: df = df.sort_values('file')
14/7:
df = df.reset_index()
df = df.drop( columns="index")
14/8: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
14/9: imprt numpy as np
14/10: import numpy as np
14/11: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
14/12: df['pos'] = df['conf'] >= 0.8
14/13:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
14/14: res.sort_values("pos")
14/15: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_minute.pdf")
14/16:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
14/17: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_heure.pdf")
14/18: %history -f save-ipython/export_results_V3.txt
15/1:
import pandas as pd
import glob
import matplotlib.pyplot as plt
15/2: df = pd.DataFrame(columns = ['file','idx','espece','x','y','w','h','conf'])
15/3: files = glob.glob('yolov5/runs/detect/exp_BERMUDE_20210224_20210326/labels/*.txt')
15/4:
for f in files:
    tab = pd.read_csv(f, sep=' ', names = ['espece','x','y','w','h','conf'])
    tab['file'] = f
    tab['idx'] = int(f.split('-')[-1][:-4])
    df = pd.concat([df, tab])
df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
df = df.sort_values('file')
df = df.reset_index()
df = df.drop( columns="index")
15/5: import numpy as np
15/6: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
15/7: plt.close(); plt.figure(figsize=(16,9));plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_BERMUDE_20210224_20210326.pdf")
15/8: df['pos'] = df['conf'] >= 0.8
15/9:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
res.sort_values("pos")
15/10:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
15/11: res
15/12: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection BERMUDE_20210224_20210326"); plt.savefig("detection_BERMUDE_20210224_20210326_minute.pdf")
15/13: res
15/14:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
15/15: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection BERMUDE_20210224_20210326"); plt.savefig("detection_BERMUDE_20210224_20210326_heure.pdf")
15/16:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("D")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
15/17: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection BERMUDE_20210224_20210326"); plt.savefig("detection_BERMUDE_20210224_20210326_day.pdf")
15/18: res.sort_values("pos")
15/19:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
15/20: res.sort_values("pos")
15/21:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
15/22: res.sort_values("pos")
15/23: ls
15/24: files
16/1:
import pandas as pd
import glob
import matplotlib.pyplot as plt
import numpy as np
16/2: df = pd.DataFrame(columns = ['file','idx','espece','x','y','w','h','conf'])
16/3: files = glob.glob('yolov5/runs/detect/exp_GUA_AB_20210515_20210616/labels/*.txt')
16/4: files.sort()
16/5: files
16/6:
for f in files:
    tab = pd.read_csv(f, sep=' ', names = ['espece','x','y','w','h','conf'])
    tab['file'] = f
    tab['idx'] = int(f.split('-')[-1][:-4])
    df = pd.concat([df, tab])
16/7: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
16/8: df = df.sort_values('file')
16/9: df = df.reset_index()
16/10: df = df.drop( columns="index")
16/11: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
16/12: df['pos'] = df['conf'] >= 0.8
16/13: df
16/14: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.show()
16/15: plt.close();
16/16: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience.pdf")
16/17: plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_GUA_AB_20210515_20210616.pdf")
16/18: plt.close(); plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_GUA_AB_20210515_20210616.pdf")
16/19: df['pos'] = df['conf'] >= 0.2
16/20: plt.close(); plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_GUA_AB_20210515_20210616.pdf")
16/21: df['pos'] = df['conf'] >= 0.8
16/22: plt.close(); plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_GUA_AB_20210515_20210616.pdf")
16/23: df['pos'].sum()
16/24:

res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
res.sort_values("pos")
16/25: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_minute.pdf")
16/26:

res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
res.sort_values("pos")
16/27:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_heure.pdf")
16/28:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("D")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_heure.pdf")
16/29:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("D")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_jour.pdf")
16/30:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection Fjord_04_11_2022"); plt.savefig("detection_Fjord_04_11_2022_heure.pdf")
16/31:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection GUA_AB_20210515_20210616"); plt.savefig("detection_GUA_AB_20210515_20210616_heure.pdf")
16/32:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("D")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(res["pos"])), res["pos"]); plt.xticks(np.arange(len(res["date"])), res["date"].dt.strftime("%Y-%m-%d")); plt.xticks(rotation=90, fontsize=7); plt.title("detection GUA_AB_20210515_20210616"); plt.savefig("detection_GUA_AB_20210515_20210616_jour.pdf")
16/33: sys.args
16/34: import os
16/35: import sys
16/36: sys.args
16/37: sys.arg
16/38: sys.argv
16/39: files
16/40: pwd
16/41: %run export_results_V3.py GUA_AB_20210515_20210616
16/42: %run export_results_V3.py GUA_AB_20210515_20210616
16/43: %run export_results_V3.py GUA_AB_20210515_20210616
16/44: %run export_results_V3.py GUA_AB_20210515_20210616
16/45: %run export_results_V3.py GUA_AB_20210515_20210616
16/46: %run export_results_V3.py GUA_AB_20210515_20210616
16/47: %run export_results_V3.py GUA_AB_20210515_20210616
16/48: name_sess
16/49: %run export_results_V3.py GUA_AB_20210515_20210616
16/50: plt.close(); plt.figure(figsize=(16,9));plt.hist(df['conf'], bins=np.arange(0,101)/100);plt.savefig("hist_confience_%s.pdf"%(name_sess))
16/51: %run export_results_V3.py GUA_AB_20210515_20210616
16/52: %run export_results_V3.py GUA_AB_20210412_20210512
16/53: %run export_results_V3.py GUA_AB_20210211_20210316
16/54: %run export_results_V3.py BERMUDE_20210411_20210511
17/1: %run export_results_V3.py BERMUDE_20210224_20210326
16/55: %run export_results_V3.py Fjord3D_041122
16/56: %run export_results_V3.py Fjord3D_041122
16/57: %run export_results_V3.py Fjord3D_051122
17/2: %run export_results_V3.py BERMUDE_20210224_20210326
17/3: df
17/4: res
17/5: %run export_results_V3.py BERMUDE_20210224_20210326
17/6:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
17/7: res
17/8: res.sort_values("pos")
17/9: res = res.sort_values("pos")
17/10: res[res['date']== "20350310"]
17/11: res
17/12: res[res['date']=="2035-03-10"]
17/13: res[res['date'].dt.floor("D")=="2035-03-10"]
17/14: res[res['date'].dt.floor("D")=="2035-03-10"]
17/15: res
17/16: res = res.sort_values("date")
17/17: res
17/18: res[res['date'].dt.floor("D")=="2035-03-10"]
17/19: plt.plot(res['pos']);plt.show()
17/20: plt.plot(res['pos']);plt.show()
17/21: plt.plot(res['date'], res['pos']);plt.show()
17/22: res
17/23: plt.plot(res['date'], res['pos']);plt.show()
17/24: tmp = res[res['date'].dt.floor("D")=="2035-03-10"]
17/25: plt.plot(tmp['date'], tmp['pos']);plt.show()
17/26: tmp
17/27: tmp.iloc[2027]
17/28: tmp.iloc[20]
17/29: tmp.iloc[90]
17/30: tmp.iloc[120]
17/31: tmp.iloc[123]
17/32: tmp.iloc[125]
17/33: tmp.iloc[121]
17/34: tmp.iloc[122]
17/35: tmp.iloc[123]
17/36: tmp.iloc[124]
17/37: tmp.iloc[125]
17/38: tmp.iloc[126]
17/39: tmp.iloc[125]
17/40: tmp.iloc[124]
17/41: tmp.iloc[123]
17/42: tmp.iloc[124]
17/43: tmp.iloc[125]
17/44: tmp.iloc[126]
17/45: tmp.iloc[127]
17/46: tmp.iloc[128]
17/47: tmp.iloc[129]
17/48: tmp.iloc[130]
17/49: tmp
17/50: tmp.iloc[120:130]
17/51: df
17/52: res
17/53: res.iloc[4000]
17/54: res.iloc[400]
17/55: res.iloc[3000]
17/56: res.iloc[2500]
17/57: res.iloc[2200]
17/58: res.iloc[2000]
17/59: res.iloc[2050]
17/60: tmp
17/61: res.iloc[2100]
17/62: res.iloc[2150]
17/63: res.iloc[2151]
17/64: res.iloc[2152]
17/65: res.iloc[2153]
17/66: res.iloc[2154]
17/67: res.iloc[2155]
17/68: res.iloc[2156]
17/69: res
17/70: df[df['date'].dt.floor('H')=="2035-03-10 17:00:00"]
17/71: df[df['date'].dt.floor('min')=="2035-03-10 17:11:00"]
17/72: df[df['date'].dt.floor('min')=="2035-03-10 18:11:00"]
17/73: df[df['date'].dt.floor('min')=="2035-03-10 17:23:00"]
17/74: df[df['date'].dt.floor('min')=="2035-03-10 17:23:00"].sum()
17/75: df[df['date'].dt.floor('min')=="2035-03-10 17:23:00"]['pos'].sum()
17/76: df[df['date'].dt.floor('min')=="2035-03-10 17:23:00"]['pos']
17/77: df[df['date'].dt.floor('min')=="2035-03-10 17:23:00"]['pos'].sum()
17/78: df[df['date'].dt.floor('min')=="2035-03-10 17:11:00"]['pos'].sum()
17/79: df[df['date'].dt.floor('min')=="2035-03-10 17:35:00"]['pos'].sum()
17/80: df[df['date'].dt.floor('min')=="2035-03-10 17:35:00"]['pos']
17/81: df[df['date'].dt.floor('min')=="2035-03-10 17:35:00"]
17/82: df[df['date'].dt.floor('min')=="2035-03-10 17:41:00"]
17/83: df['date'].dt.floor('min')=="2035-03-10 17:41:00
17/84: df['pos'] = np.logical_and(df['conf'] >= 0.6, df['conf'] <=0.8)
17/85: df
17/86:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
17/87: res
17/88: res.sort_values("pos")
17/89: plt.plot(tmp['date'], tmp['pos']);plt.show()
17/90: plt.plot(res['date'], res['pos']);plt.show()
17/91:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
17/92: plt.plot(res['date'], res['pos']);plt.show()
17/93:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("min")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
17/94: plt.plot(res['date'], res['pos']);plt.show()
17/95: res.sort_values("pos")
17/96: res.sort_values("pos")[:-10]
17/97: res.sort_values("pos")[-10:]
17/98: files
17/99: file
17/100: res.sort_values("pos")[-10:]
17/101: res.sort_values("pos")[-20:-10]
20/1: %run yolo-dyni/labelme2yolo.py Spectrogram_anot/ -o Labels_anot/
20/2: %run yolo-dyni/get_train_val_YOLO.py ./Spectrogram_anot/ ./Labels_anot/ train_2023_03_02
21/1: import pandas as pd
21/2: df = pd.read_pickle("/nfs/NAS6/SABIOD/SITE/CARIMAM/whale_detector/manip_stephane/predictions_sans_classification_04_05.pkl")
21/3: df
21/4: df['sess'] = df['fn'].str.split('/', extend=True)[0]
21/5: df['sess'] = df['fn'].str.split('/', extendd=True)[0]
21/6: df['sess'] = df['fn'].str.split('/', extends=True)[0]
21/7: df['sess'] = df['fn'].str.split('/', expend=True)[0]
21/8: df['sess'] = df['fn'].str.split('/', expand=True)[0]
21/9: df
21/10: df['lot'] = df['fn'].str.split('/', expand=True)[0]
21/11: df['sess'] = df['fn'].str.split('/', expand=True)[1].str.split('_')[0]
21/12: df
21/13: df['fn'].str.split('/', expand=True)[1].str.split('_')[0]
22/1: %run export_results_V3.py GUA_AB_20210211_20210316
23/1: %run export_results_V3.py 8
23/2: %run export_results_V3.py "8"
23/3: %run export_results_V3.py "8"
23/4: %run export_results_V3.py "1"
23/5: %run export_results_V3.py "1"
23/6: %run export_results_V3.py "2"
24/1: %run export_results_V3.py "3"
25/1: %run export_results_V3.py "4"
25/2: %run export_results_V3.py "5"
25/3: %run export_results_V3.py "6"
24/2: %run export_results_V3.py "7"
23/7: %run export_results_V3.py "8"
24/3: xit
25/4: df
25/5: df['conf']
25/6: df['conf']
25/7: df
25/8: df
25/9: df_pos = df[df['pos']== True]
25/10: df_pos
25/11: df_pos['heure'] = df_pos['date'].dt.hour
25/12:  df_pos['date'].dt.hour
25/13: df_pos
25/14: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(df_pos['heure']), df_pos['heure']));  plt.xticks(rotation=90, fontsize=7); plt.title("detection %s"%(name_sess)); plt.show()
25/15: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(df_pos['heure'])), df_pos['heure']);  plt.xticks(rotation=90, fontsize=7); plt.title("detection %s"%(name_sess)); plt.show()
25/16: df_pos['heure']
25/17: df_pos
25/18: res = list()
25/19:
for d, elem in df.groupby(df['heure']):
    break
25/20: df
25/21:
for d, elem in df.groupby(df_pos['heure']):
    break
25/22: elem
25/23: print(d)
25/24: elem.sum()
25/25: len(elem)
25/26:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*1))
tab_res *= -1 
for idx, row in df.iterrows():
    df['pos'] == True:
        tab_res[(row.date - df['date'].min()).days, row.date.hour*1] += row['pos']
25/27:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*1))
tab_res *= -1 
for idx, row in df.iterrows():
    if df['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*1] += row['pos']
25/28:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*1))
tab_res *= -1 
for idx, row in df.iterrows():
    if row['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*1] += row['pos']
25/29: tab_res
25/30: plt.imshow(tab_res);plt.show()
25/31: plt.imshow(tab_res.T, aspect='auto', interpolation=None);plt.show()
25/32: plt.imshow(tab_res.T, aspect='auto', interpolation=None);plt.colorbar(); plt.show()
25/33:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*6))
tab_res *= -1 
for idx, row in df.iterrows():
    if row['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*6+ row.date.minute//10] += row['pos']
25/34: plt.imshow(tab_res.T, aspect='auto', interpolation=None);plt.colorbar(); plt.show()
25/35:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*2))
tab_res *= -1 
for idx, row in df.iterrows():
    if row['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*2+ row.date.minute//30] += row['pos']
25/36: plt.imshow(tab_res.T, aspect='auto', interpolation=None);plt.colorbar(); plt.show()
25/37: plt.imshow(tab_res.T, aspect='auto', interpolation=None, origin="lower");plt.colorbar(); plt.show()
25/38:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*1))
tab_res *= -1 
for idx, row in df.iterrows():
    if row['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*1+ row.date.minute//60] += row['pos']
25/39: plt.imshow(tab_res.T, aspect='auto', interpolation=None, origin="lower");plt.colorbar(); plt.show()
25/40: tab_res.max()
25/41: tab_res.argmax()
25/42: tab_res.imax()
25/43: tab_res.max()
25/44: tab_res
25/45: tab_res.argmax(axis=1)
25/46: tab_res.argmax(axis=0)
25/47: tab_res.max(axis=0)
25/48: ls
25/49: plt.imshow(tab_res.T, aspect='auto', interpolation=None, origin="lower");plt.colorbar(); plt.show()
25/50: ls
25/51: df
25/52:
res = list()
for d, elem in df.groupby(df['date'].dt.floor("H")):
    res.append([d, elem['pos'].sum()])
res = pd.DataFrame(res, columns=["date", "pos"])
25/53: res
25/54: res.pos.max()
25/55: res[res.pos==res.pos.max()]
25/56:
nb_day = (df['date'].max() - df['date'].min()).days + 1
tab_res = np.ones((nb_day, 24*1))
tab_res *= -1 
for idx, row in df.iterrows():
    if row['pos'] == True :
        tab_res[(row.date - df['date'].min()).days, row.date.hour*1+ row.date.minute//60] += row['pos']
25/57: plt.imshow(tab_res.T, aspect='auto', interpolation=None, origin="lower");plt.colorbar(); plt.show()
26/1: import torch
26/2: torch.__version__
27/1: %run  train_detect.py  --data_path /short/CARIMAM/DATA/ --df_path Annotation_CARIMAM_apo_22_10_02.xlsx --ne 40 --device 1 --bs 32  --hp --aug
28/1: %run  train_detect.py  --data_path /short/CARIMAM/DATA/ --df_path Annotation_CARIMAM_apo_22_10_02.xlsx --ne 40 --device 1 --bs 32  --hp --aug
31/1: %run train_both_CNN.py --data_path /short/CARIMAM/DATA/ --df_path Annotation_CARIMAM_apo_22_10_02.xlsx --device 1 --weight detector_carimam/03-14-23_13\:20\:40\:hp\=True\:lr\=0.0003\:ne\=5\:wd\=0.05\:bs\=32\:blcd\=False\:aug\=True/ckpt_3584.pth --hp --aug
137/1:
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
137/2: plt.show()
137/3:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
137/4: plt.close(); plt.figure(figsize=(16,9));plt.bar(np.arange(len(df_pos['heure'])), df_pos['heure']);  plt.xticks(rotation=90, fontsize=7); plt.title("detection %s"%(name_sess)); plt.show()
137/5: plt.show()
138/1:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
138/2: plt.show()
143/1: ls
143/2: cd LOT9/
143/3: ls
143/4: cd RivCAYENNE_GUYANE_GEPOG_20210626_20210804/
143/5: cd ..
143/6: ls
143/7: cd ..
143/8: ls
143/9: cd LOT2_GUYANNELOT2/
143/10: ls
143/11: cd GUYANNE_GEPOG_20210313_20210419/
143/12: ls
143/13: cd '*.WAV'
143/14: ls
143/15: ls
143/16: cd ..
143/17: cd ..
143/18: cd ..
143/19: ls
143/20: mv LOT2_GUYANNELOT2/ LOT2/
143/21: ls
143/22: cd LOT2
143/23: ls
143/24: cd GUYANNE_GEPOG_20210313_20210419/
143/25: ls
143/26: mv '*.WAV' .
143/27: mv "*.WAV/" .
143/28: mv "*.WAV/" RES
143/29: ls
143/30: ls
143/31: mv RES/ .
143/32: mv RES/* .
144/1: %run compute_quantile.py
145/1: %run compute_quantile.py
145/2: df = pd.read_pickle("results_all.pkl")
145/3: sd
145/4: df
145/5: %run compute_quantile.py
145/6: %run compute_quantile.py
145/7: %run compute_quantile.py
145/8: df = pd.read_pickle("results_all.pkl")
145/9: df
145/10: %run compute_quantile.py
145/11: %run compute_quantile.py
143/33: ls
143/34:  cd ..
145/12: ls
145/13: rm results_all.pkl
145/14: %run compute_quantile.py
145/15: %run compute_quantile.py
146/1: df = pd.read_pickle("results_all.pkl")
146/2: import pandas as pd
146/3: df
146/4: df = pd.read_pickle("results_all.pkl")
146/5: df
146/6: df.sort_values('nb_pos')
146/7: df.sort_values('nb_pos').iloc[-1]
146/8: df.sort_values('nb_pos').iloc[-1].file
146/9: df.sort_values('nb_pos').iloc[-10:].file
146/10: df.sort_values('nb_pos').iloc[-2].file
146/11: df.sort_values('nb_pos').iloc[-3].file
146/12: df.sort_values('nb_pos')
146/13: df.sort_values('nb_pos').nb_pos
146/14: plt.plot(df.sort_values('nb_pos').nb_pos);plt.show()
146/15: import matplotlib.pyplot as plt
146/16: plt.plot(df.sort_values('nb_pos').nb_pos);plt.show()
146/17: plt.plot(df.sort_values('nb_pos').nb_pos, "*");plt.show()
146/18: df.sort_values('nb_pos')
146/19: df.sort_values('nb_pos').nb_pos
146/20: plt.plot(np.array(df.sort_values('nb_pos').nb_pos, "*"));plt.show()
146/21: import numpy as np
146/22: plt.plot(np.array(df.sort_values('nb_pos').nb_pos, "*"));plt.show()
146/23: plt.plot(np.array(df.sort_values('nb_pos').nb_pos), "*");plt.show()
146/24: df.sort_values('nb_pos')
146/25: df.sort_values('nb_pos')[-2]
146/26: df.sort_values('nb_pos').iloc[-2]
146/27: df.sort_values('nb_pos').iloc[-2].file
146/28: pred = np.load(df.sort_values('nb_pos').iloc[-2].file)['arr_0']
146/29: plt.plot(pred);plt.show()
146/30: pred = np.load(df.sort_values('nb_pos').iloc[-1].file)['arr_0']
146/31: plt.plot(pred);plt.show()
146/32: pred = np.load(df.sort_values('nb_pos').iloc[-5].file)['arr_0']
146/33: plt.plot(pred);plt.show()
146/34: df.sort_values('nb_pos').iloc[-5].file
169/1: 2048/8
190/1: df = pd.read_pickle("results_all.pkl")
190/2: import pandas as pd
190/3: df = pd.read_pickle("results_all.pkl")
190/4: df
191/1: import pandas as pd
191/2: df = pd.read_pickle("results_all.pkl")
191/3: df
191/4: plt.plot(df.nb_pos);plt.show()
191/5: import matplotlib.pyplot as plt
191/6: plt.plot(df.nb_pos);plt.show()
191/7: df.sort_values('nb_pos').iloc[-5].file
191/8: df.sort_values('nb_pos').iloc[:-5].file
191/9: df.sort_values('nb_pos').iloc[:-1].file
191/10: df.sort_values('nb_pos').iloc[-1].file
191/11: df.sort_values('nb_pos').iloc[-1]
191/12: df.sort_values('nb_pos').iloc[-1].files
191/13: df.sort_values('nb_pos').iloc[-5].file
191/14: df.sort_values('nb_pos').iloc[-10].file
191/15: df.sort_values('nb_pos')
191/16: df.sort_values('nb_pos').iloc[-100].nb_pos
191/17: df.sort_values('nb_pos').iloc[-50].nb_pos
191/18: df.sort_values('nb_pos').iloc[-20].nb_pos
191/19: df.sort_values('nb_pos').iloc[-20].file
191/20: df.sort_values('nb_pos').iloc[-22].file
191/21: df.sort_values('nb_pos').iloc[-22].nb_pos
191/22: df.sort_values('nb_pos').iloc[-22].file
191/23: df.sort_values('nb_pos').iloc[-16].file
191/24: df.sort_values('nb_pos').iloc[-14].file
191/25: df.sort_values('nb_pos').iloc[-12].file
191/26: df.sort_values('nb_pos').iloc[-8].file
191/27: plt.plot(df.nb_pos);plt.show()
191/28: df
191/29:     files = glob.glob('PORT*/*_pred.npz')
191/30: import glob
191/31:     files = glob.glob('PORT*/*_pred.npz')
191/32:     pred = np.load(filename)['arr_0']
191/33: import numpy as np
191/34:     pred = np.load(filename)['arr_0']
191/35:     pred = np.load(filenames[0])['arr_0']
191/36:     pred = np.load(files[0])['arr_0']
191/37: pred
191/38: files[0]
191/39: pos = np.load("PORTNOUVELLE_2023_05_03/20220703_044941UTC_V12.log.wav_pos_sequence.npz")['arr_0']
191/40: pos
191/41: pos/60
191/42: pred
191/43: %run compute_quantile.py
191/44: tmp = pd.read_csv("results_all.csv")
191/45: %run compute_quantile.py
191/46: tmp = pd.read_csv("results_all.csv")
191/47: tmp
191/48: tmp.iloc[0].prediction
191/49: %run compute_quantile.py
191/50: %run compute_quantile.py
191/51: tmp = pd.read_pkl("results_all.pkl")
191/52: tmp = pd.read_pickle("results_all.pkl")
191/53: tmp
203/1: imprt numpy as np
203/2: import numpy as np
203/3: import pandas as pd
203/4: df = pd.read_pickle("results_all.pkl")
203/5: df
203/6: df.sort_values('nb_pos')
203/7: df.sort_values('nb_positif')
203/8: df = pd.read_pickle("results_all.pkl")
203/9: df.sort_values('nb_positif')[-1].nb_positif
203/10: df.sort_values('nb_positif').iloc[-1].nb_positif
203/11: df.sort_values('nb_positif').iloc[-1].file
203/12: plt.plot(df.sort_values('nb_positif').iloc[-1].prediction);plt.show()
203/13: import matplotlib.pyplot as plt
203/14: plt.plot(df.sort_values('nb_positif').iloc[-1].prediction);plt.show()
203/15: plt.plot(df.sort_values('nb_positif').iloc[-10].prediction);plt.show()
203/16: plt.plot(df.sort_values('nb_positif').iloc[-10].prediction); plt.title(df.sort_values('nb_positif').iloc[-10].file);plt.show()
203/17: print(df.sort_values('nb_positif').iloc[-10].file); plt.plot(df.sort_values('nb_positif').iloc[-10].prediction);plt.show()
203/18: print(df.sort_values('nb_positif').iloc[-5].file); plt.plot(df.sort_values('nb_positif').iloc[-5].prediction);plt.show()
203/19: print(df.sort_values('nb_positif').iloc[-5].file); plt.plot(df.sort_values('nb_positif').iloc[-5].prediction);plt.show()
203/20: %run compute_quantile.py
203/21: %run compute_quantile.py
203/22: df = pd.read_pickle("results_all.pkl")
203/23: df
203/24: df.sort_values('nb_positif')
203/25: df.sort_values('nb_positif').iloc[-1].nb_positif
203/26: df.sort_values('nb_positif').iloc[-2].file
203/27: df.sort_values('nb_positif').iloc[-5].file
203/28: df.sort_values('nb_positif').iloc[-5].nb_positif
203/29: df.sort_values('nb_positif').iloc[-15].nb_positif
203/30: df.sort_values('nb_positif').iloc[-20].nb_positif
203/31: df.sort_values('nb_positif').iloc[-25].nb_positif
203/32: df.sort_values('nb_positif').iloc[-25].file
203/33: df.sort_values('nb_positif').iloc[-30].file
203/34: df.sort_values('nb_positif').iloc[-30].nb_positif
203/35: df.sort_values('nb_positif').iloc[-30].nb_positif
203/36: df.sort_values('nb_positif').iloc[-30].file
203/37: df.sort_values('nb_positif').iloc[-35].file
210/1: pd
210/2: import pandas as pd
210/3: df = pd.read_pickle("results_all.pkl")
210/4: df
210/5: tmp = df.drop(columns="prediction")
210/6: tmp.to_csv("results_all.csv")
210/7: tmp
210/8: pwd
210/9: tmp
210/10: tmp = tmp.file.str.replace("_pred.npz", "")
210/11: tmp
210/12: tmp = df.drop(columns="prediction")
210/13: tmp
210/14: tmp.file.str.replace("_pred.npz", "")
210/15: tmp
210/16: tmp.file.iloc[0]
210/17: tmp.file = tmp.file.str.replace("_pred.npz", "")
210/18: tmp.file.iloc[0]
210/19: tmp.to_csv("results_all.csv")
210/20: df.file =  df.file.str.replace("_pred.npz", "")
210/21: df.file.iloc[0]
210/22: df.file.apply( lambda x : os.path.basename(x))
210/23: import os
210/24: df.file = df.file.apply( lambda x : os.path.basename(x))
210/25: df.file.iloc[0]
210/26: tmp = df.drop(columns="prediction")
210/27: tmp.to_csv("results_all.csv")
210/28: df
210/29: tmp = df.drop(columns=["nb_positif", "q80", "q90", "q95", "q99"])
210/30: tmp
210/31: tmp.to_csv("results_pred_all.csv")
220/1: import pandas as pd
220/2: df = pd.read_pickle("results_all.pkl")
220/3: df
220/4: df.sort_values('nb_positif')
220/5: df.sort_values('nb_positif').iloc[-1].nb_positif
220/6: df.sort_values('nb_positif').iloc[-10].nb_positif
220/7: df.sort_values('nb_positif').iloc[-20].nb_positif
220/8: df.sort_values('nb_positif').iloc[-20].file
220/9: df.sort_values('nb_positif').iloc[-1].file
220/10: df.sort_values('nb_positif').iloc[-15].file
220/11: df.sort_values('nb_positif').iloc[-50].file
220/12: df.sort_values('nb_positif').iloc[-50].nb_positif
220/13: df.sort_values('nb_positif').iloc[-30].file
220/14: df.sort_values('nb_positif').iloc[-30].nb_p
220/15: df.sort_values('nb_positif').iloc[-30].nb_positif
220/16: df.sort_values('nb_positif').iloc[-25].nb_positif
220/17: df.sort_values('nb_positif').iloc[-25].file
220/18: tmp = df.sort_values('nb_positif').iloc[:-10].file
220/19: tmp
220/20: tmp.to_csv("false_detect.csv")
220/21: pwd
220/22: df = pd.read_pickle("PORTNOUVELLE/20220623_20220812/results_all.pkl")
220/23: df.sort_values('nb_positif').iloc[-25].file
220/24: tmp = df.sort_values('nb_positif').iloc[:-10].file
220/25: tmp.to_csv("false_detect_20220623.csv")
220/26: df = pd.read_pickle("results_all_LOT9_GUY.pkl")
220/27: df
220/28: df.sort_values('nb_positif').iloc[-10].file
220/29: tmp = df.sort_values('nb_positif').iloc[:-10].file
220/30: tmp.to_csv("false_detect_LOT9.csv")
220/31: %run compute_quantile.py
220/32: %run compute_quantile.py
220/33: df = pd.read_pickle("results_all.pkl")
220/34: df
220/35: df.sort_values('nb_positif').nb_positif
220/36: df.sort_values('nb_positif').iloc[-5].file
220/37: df.sort_values('nb_positif').iloc[-5].nb_positif
220/38: df.sort_values('nb_positif').iloc[-3].file
220/39: %run compute_quantile.py
220/40: df = pd.read_pickle("results_all.pkl")
220/41: df
220/42: df = pd.read_pickle("results_all.pkl")
220/43: df = pd.read_pickle("results_all.pkl")
220/44: df
220/45: %run compute_quantile.py
220/46: df
220/47: df.sort_values('nb_positif').nb_positif
220/48: df = pd.read_pickle("results_all_JAM.pkl")
220/49: df.sort_values('nb_positif').nb_positif
220/50: df.sort_values('nb_positif').iloc[-3].file
220/51: df.sort_values('nb_positif').iloc[-5].file
220/52: df.sort_values('nb_positif').iloc[-10].file
220/53: df.sort_values('nb_positif').iloc[-20].file
220/54: df.sort_values('nb_positif').iloc[-25].file
220/55: df.sort_values('nb_positif').iloc[-26].file
220/56: df.sort_values('nb_positif').iloc[-30].file
220/57: %run compute_quantile.py
220/58: df
220/59: df = pd.read_pickle("results_all_ANG.pkl")
220/60: df
220/61: df.sort_values('nb_positif').nb_positif
220/62: df = df.dropna()
220/63: df.sort_values('nb_positif').nb_positif
220/64: df.sort_values('nb_positif').iloc[-5].file
220/65: df.sort_values('nb_positif').iloc[-15].file
220/66: df.sort_values('nb_positif').iloc[-15].nb_positif
220/67: df.sort_values('nb_positif').iloc[-25].nb_positif
220/68: df.sort_values('nb_positif').iloc[-25].file
220/69: %run compute_quantile.py
220/70: %run compute_quantile.py
221/1: import pandas as pd
221/2: df = pd.read_pickle("results_all.pkl")
221/3: df
221/4: df = pd.read_pickle("PORTNOUVELLE/20220623_20220812/results_all.pkl")
221/5: df
221/6: df.sort_values('nb_positif').
221/7: df.sort_values('nb_positif')
221/8: tmp = df.sort_values('nb_positif')[:-10]
221/9: tmp
221/10: tmp = df.sort_values('nb_positif')
221/11: tmp = df.sort_values('nb_positif')[-10:]
221/12: tmp
221/13: tmp.shape
221/14: tmp.to_csv("false_detect_EOLBIO.csv")
221/15: df = pd.read_pickle("PORTNOUVELLE/202209*/results_all.pkl")
221/16: df = pd.read_pickle("PORTNOUVELLE/20220901_20221021/results_all.pkl")
222/1: %run compute_quantile.py
222/2: df = pd.read_pickle("results_all_GUY.pkl")
222/3: df
222/4: df.sort_values('nb_positif')
222/5: tmp = df.sort_values('nb_positif').iloc[:-10].file
222/6: tmp
222/7: tmp.shape
222/8: tmp = df.sort_values('nb_positif').iloc[-10:].file
222/9: tmp
222/10: tmp.shaope
222/11: tmp.shape
222/12: tmp.to_csv("false_detect_GUY.csv")
220/71: %run compute_quantile.py
223/1: %run compute_quantile.py
222/13: df = pd.read_pickle("results_all_BON.pkl")
222/14: df
222/15: df.sort_values('nb_positif')
222/16: df = df.dropna()
222/17: df
222/18: df.sort_values('nb_positif')
222/19: df.sort_values('nb_positif').iloc[-10:].file
222/20: tmp = df.sort_values('nb_positif').iloc[-10:].file
222/21: tmp.shape
222/22: tmp.to_csv("false_detect_BON.csv")
222/23: %run compute_quantile.py
222/24: df = pd.read_pickle("results_all_BERM.pkl")
222/25: df = df.dropna()
222/26: df
222/27: df.sort_values('nb_positif').iloc[-10:].file
222/28: df.sort_values('nb_positif').iloc[-10:].nb_posit
222/29: df.sort_values('nb_positif').iloc[-10:].nb_positif
222/30: df.sort_values('nb_positif').iloc[-10:].file
222/31: tmp = df.sort_values('nb_positif').iloc[-10:].file
222/32: tmp.to_csv("false_detect_BERM.csv")
222/33: df = pd.read_pickle("results_all_ANG.pkl")
222/34: df = df.dropna()
222/35: df.sort_values('nb_positif').iloc[-10:].file
222/36: df.sort_values('nb_positif').iloc[-10:].nb_positif
222/37: tmp = df.sort_values('nb_positif').iloc[-10:].file
222/38: tmp.to_csv("false_detect_ANG.csv")
222/39: %run compute_quantile.py
222/40: df = pd.read_pickle("results_all_StEUS.pkl")
222/41: df = df.dropna()
222/42: df
222/43: df.sort_values('nb_positif').iloc[-10:].nb_positif
222/44: tmp = df.sort_values('nb_positif').iloc[-10:].file
222/45: tmp.to_csv("false_detect_StEUS.csv")
237/1: np
237/2: import numpy as np
237/3: import scipy as scp
237/4: import matplotlib.pyplot as plt
237/5: pred = np.load("result_both/FALSE/out_21220302_105301UTC_V00OS11.WAV_pred.npy")
237/6: pos = np.load("result_both/FALSE/out_21220302_105301UTC_V00OS11.WAV_pos_sequence.npy")
237/7: pos
237/8: pred > 0.5
237/9: pred[:,1] > 0.5
237/10: positif = pred[:,1] > 0.5
237/11: positif
237/12: idx_pos = pred[:,1] > 0.5
237/13: pos = np.load("result_both/FALSE/out_21220302_105301UTC_V00OS11.WAV_pos_sequence.npy")
237/14: pos[idx_pos]
237/15:
anot_list = list()
for idx_f in pos[idx_pos]:
    break
237/16: idx_f
237/17: import soundfile as sf
237/18: sig, sr = sf.read("/nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/BON_20210211-131700UTC_20320321/21220302_105301UTC_V00OS11.WAV")
237/19: sig[idx_f*sr: (idx_f+0.128)*sr]
237/20: sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
237/21: plt.plot(sig[int(idx_f*sr): int((idx_f+0.128)*sr)]);plt.show()
237/22: tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
237/23:         self.hp_sos = sg.butter(3, 2500, 'hp', output='sos', fs=_FS)
237/24:         self.hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=_FS)
237/25:         self.hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=sr)
237/26: hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=sr)
237/27: tmp = sg.sosfiltfilt(hp_sos, tmp)
237/28: tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
237/29: plt.plot(tmp);plt.show()
237/30: idx_max = np.argmax(np.abs(tmp))
237/31: plt.plot(tmp);plt.plot(idx_max, tmp[idx_max], "*");plt.show()
237/32: idx_max
237/33: idx_max/sr
237/34: idx_f + (idx_max/sr)
237/35:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
237/36: anot_list
237/37: idx_pos.sum()
237/38: plt.plot(sig); plt.plot(anot_list, sig[anot_list]);plt.show()
237/39: plt.plot(sig); plt.plot(anot_list, sig[int(anot_list*sr)]);plt.show()
237/40: plt.plot(sig); plt.plot(anot_list, sig[int(np.array(anot_list)*sr)]);plt.show()
237/41: plt.plot(sig); plt.plot(anot_list, sig[(anot_list*sr).astype(int)]);plt.show()
237/42: plt.plot(sig); plt.plot(anot_list, sig[(np.array(anot_list)*sr).astype(int)]);plt.show()
237/43: plt.plot(sig); plt.plot(anot_list, sig[(np.array(anot_list)*sr).astype(int)], *);plt.show()
237/44: plt.plot(sig); plt.plot(anot_list, sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
237/45: plt.plot(sig); plt.plot(np.array(anot_list*sr).astype(), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
237/46: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
237/47: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
237/48: anot_list
237/49: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
238/1: df = pd.read_pickle("results_all_GUY.pkl")
238/2: import pandas as pd
238/3: import matplotlib.pyplot as plt
238/4: import numpy as np
238/5: df = pd.read_pickle("results_all_GUY.pkl")
238/6: df
238/7: df.sort_values('nb_positif').nb_positif
238/8: df.sort_values('nb_positif').iloc[-5:].nb_positif
238/9: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/1: import pandas as pd
239/2: import matplotlib.pyplot as plt
239/3: import numpy as np
239/4: df = pd.read_pickle("results_all_PNN.pkl")
239/5: sd
239/6: df
239/7: df.sort_values('nb_positif')
239/8: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/9: df.sort_values('nb_positif').iloc[-15:].files
239/10: df.sort_values('nb_positif').iloc[-15:].file
239/11: tmp = df.sort_values('nb_positif').iloc[-15:].file
239/12: df
239/13: nb_positif = (prediction > 0.4).sum()
239/14: nb_positif = (df.prediction > 0.4).sum()
239/15: df.nb_positif = (df.prediction > 0.4).sum()
239/16: df.prediction > 0.4
239/17: %run compute_quantile.py
239/18: df = pd.read_pickle("results_all_PNN.pkl")
239/19: df
239/20: df.sort_values('nb_positif').iloc[-15:].file
239/21: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/22: %run compute_quantile.py
239/23: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/24: df.sort_values('nb_positif').iloc[-15:].hist
239/25: df = pd.read_pickle("results_all_PNN.pkl")
239/26: df
239/27: tmp = df.sort_values('nb_positif').iloc[-15:].file
239/28: tmp.to_csv("false_detect_eolbio.csv")
237/50: %history -f MANIP_ANOT.TXT
240/1: imprt numpy as np
240/2: import numpy as np
240/3: import matplotlib.pyplot as plt
240/4: import pandas as pd
240/5: import scipy as scp
240/6: import soundfile as sf
240/7: pwd
240/8: pred = np.load("result_both/NEW_ANOT/out_20220724_040407UTC_V12.log.wav_pred.npy")
240/9: pos = np.load("result_both/NEW_ANOT/out_20220724_040407UTC_V12.log.wav_pos_sequence.npy")
240/10: pred
240/11: positif = pred[:,1] > 0.5
240/12: positif.sum()
240/13: idx_pos = pred[:,1] > 0.5
240/14: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220724_040407UTC_V12.log.wav")
240/15: hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=sr)
240/16: tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
240/17: sig = scp.signal.sosfiltfilt(hp_sos, sig)
240/18: plt.plot(sig);plt.show()
240/19:
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
240/20:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
240/21: anot_list
240/22: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/1: import soundfile as sf
241/2: import numpy as np
241/3: import scipy as scp
241/4: import matplotlib.pyplot as plt
241/5: import pa
241/6: import pandas as pd
241/7: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220623_122257UTC_V12.log.wav")
241/8: pred = np.load("result_both/NEW_ANOT_FALSE/out_20220623_122257UTC_V12.log.wav_pred.npy")
241/9: pos = np.load("result_both/NEW_ANOT_FALSE/out_20220623_122257UTC_V12.log.wav_pos_sequence.npy")
241/10: pos
241/11: positif.sum()
241/12: positif = pred[:,1] > 0.5
241/13: positif.sum()
241/14: hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=sr)
241/15: sig = scp.signal.sosfiltfilt(hp_sos, sig)
241/16:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/17: idx_pos = pred[:,1] > 0.5
241/18:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/19: anot_list
241/20: aplt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/21: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/22: np.savetxt("20220623_122257UTC_V12.log.wav.txt", anot_list)
241/23: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220716_062525UTC_V12.log.wav")
241/24: pos = np.load("result_both/NEW_ANOT_FALSE/out_20220716_062525UTC_V12.log.wav_pos_sequence.npy")
241/25: pred = np.load("result_both/NEW_ANOT_FALSE/out_20220716_062525UTC_V12.log.wav_pred.npy")
241/26: idx_pos = pred[:,1] > 0.5
241/27: sig = scp.signal.sosfiltfilt(hp_sos, sig)
241/28:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/29: anot_list
241/30: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/31: np.savetxt("20220716.txt", anot_list)
241/32: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220728_133602UTC_V12.log.wav")
241/33: pos = np.load("result_both/NEW_ANOT_FALSE/out_20220728_133602UTC_V12.log.wav_pos_sequence.npy")
241/34: pred = np.load("result_both/NEW_ANOT_FALSE/out_20220728_133602UTC_V12.log.wav_pred.npy")
241/35: sig = scp.signal.sosfiltfilt(hp_sos, sig)
241/36:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/37: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/38: np.savetxt("20220728.txt", anot_list)
241/39: anot_list
241/40: np.savetxt("20220728.txt", anot_list, %.3f)
241/41: np.savetxt("20220728.txt", anot_list, fmt=%.3f)
241/42: np.savetxt("20220728.txt", anot_list, fmt="%.3f")
241/43: pos = np.load("result_both/NEW_ANOT_FALSE/out_20220716_062525UTC_V12.log.wav_pos_sequence.npy")
241/44: pred = np.load("result_both/NEW_ANOT_FALSE/out_20220716_062525UTC_V12.log.wav_pred.npy")
241/45: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220716_062525UTC_V12.log.wav")
241/46: sig = scp.signal.sosfiltfilt(hp_sos, sig)
241/47: idx_pos = pred[:,1] > 0.5
241/48:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/49: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
241/50: np.savetxt("20220716.txt", anot_list, fmt="%.3f")
241/51: sig, sr = sf.read("/nfs/NAS3/SABIOD/SITE/PORTNOUVELLE/20220623_20220812/20220623_122257UTC_V12.log.wav")
241/52: pos = np.load("result_both/NEW_ANOT_FALSE/out_20220623_122257UTC_V12.log.wav_pos_sequence.npy")
241/53: pred = np.load("result_both/NEW_ANOT_FALSE/out_20220623_122257UTC_V12.log.wav_pred.npy")
241/54: idx_pos = pred[:,1] > 0.5
241/55:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
241/56: np.savetxt("20220623.txt", anot_list, fmt="%.3f")
241/57: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
239/29: %run compute_quantile.py
239/30: df = pd.read_pickle("results_all_PNN.pkl")
239/31: df
239/32: %run compute_quantile.py
239/33: df = pd.read_pickle("results_all_PNN.pkl")
239/34: df
239/35: df.sort_values('nb_positif').iloc[-15:].hist
239/36: tmp = df.sort_values('nb_positif').iloc[-15:].file
239/37: tmp.to_csv("false_detect_FDF.csv")
239/38: %run compute_quantile.py
239/39: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/40: tmp = df.sort_values('nb_positif').iloc[-15:].file
239/41: tmp.to_csv("false_detect_FDF.csv")
239/42: df = pd.read_pickle("results_all_GUY.pkl")
239/43: df
239/44: df.sort_values('nb_positif').iloc[-15:].nb_positif
239/45: tmp = df.sort_values('nb_positif').iloc[-10:].file
239/46: tmp.to_csv("false_detect_GUY.csv")
242/1: import pandas as pd
242/2: import soundfile as sf
242/3: import numpy as np
242/4: import scipy as scp
242/5: pos = np.load("result_both/NEW_ANOT_FALSE/out_20210406_104000UTC_V00OS11.WAV_pos_sequence.npy")
242/6: pred = np.load("result_both/NEW_ANOT_FALSE/out_20210406_104000UTC_V00OS11.WAV_pred.npy")
242/7: idx_pos = pred[:,1] > 0.5
242/8: idx_pos.sum()
242/9:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
242/10: sig, sr = sf.read("/nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUYANNE_GEPOG_20210313_20210419/20210406_104000UTC_V00OS11.WAV")
242/11: hp_sos = scp.signal.butter(3, 2500, 'hp', output='sos', fs=sr)
242/12: sig = scp.signal.sosfiltfilt(hp_sos, sig)
242/13:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
242/14: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
242/15: import matplotlib.pyplot as plt
242/16: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
242/17: np.savetxt("20210406_104000UTC_V00OS11.WAV.txt", anot_list, fmt="%.3f")
242/18: sig, sr = sf.read("/nfs/NAS6/SABIOD/SITE/CARIMAM/DATA/LOT2/GUYANNE_GEPOG_20210313_20210419/20210319_082000UTC_V00OS11.WAV")
242/19: pos = np.load("result_both/NEW_ANOT_FALSE/out_20210319_082000UTC_V00OS11.WAV_pos_sequence.npy")
242/20: pred = np.load("result_both/NEW_ANOT_FALSE/out_20210319_082000UTC_V00OS11.WAV_pred.npy")
242/21: idx_pos = pred[:,1] > 0.5
242/22: idx_pos.sum()
242/23: sig = scp.signal.sosfiltfilt(hp_sos, sig)
242/24:
anot_list = list()
for idx_f in pos[idx_pos]:
    tmp = sig[int(idx_f*sr): int((idx_f+0.128)*sr)]
    tmp = scp.signal.sosfiltfilt(hp_sos, tmp)
    idx_max = np.argmax(np.abs(tmp))
    anot_list.append(idx_f + (idx_max/sr))
242/25: plt.plot(sig); plt.plot((np.array(anot_list)*sr).astype(int), sig[(np.array(anot_list)*sr).astype(int)], "*");plt.show()
242/26: np.savetxt("20210319_082000UTC_V00OS11.WAV.txt", anot_list, fmt="%.3f")
242/27: pwd
242/28: exi
247/1: import matplotlib.pyplot as plt
247/2: import pandas as pd
247/3: df = pd.read_pickle("results_all_BERM.pkl")
247/4: df
247/5: df = df.dropna()
247/6: d
247/7: df
247/8: df['file'].str.split('/', expand=True)[1].str.split('_')[0]
247/9: df['file'].str.split('/', expand=True)[1].str.split('_')
247/10: df['file'].str.split('/', expand=True)[1].str.split('_')[0]
247/11: df['file'].str.split('/', expand=True)[1].str.split('_')[0][0]
247/12: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_')[0]
247/13: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_')
247/14: df['sess']
247/15: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_')[0]
247/16: df['file'].str.split('/', expand=True)[0]
247/17: df['lot'] = df['file'].str.split('/', expand=True)[0]
247/18: df['lot']
247/19: df['file'].str.split('/', expand=True)[0]
247/20: df['file'].str.split('/', expand=True)[-1]
247/21: df['file'].str.split('/', expand=True)[2]
247/22: df['file'].str.split('/', expand=True)[1]
247/23: df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
247/24: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
247/25: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/26: pd.to_datetime(df['file'].str.split('/', expand=True)[5].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/27: df['file']
247/28: pd.to_datetime(df['file'].str.split('/', expand=True)[3].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/29: pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/30: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/31: df
247/32:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*6))
    tab_res *= -1 
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*6+row.date.minute//10] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/33: import numpy as np
247/34:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*6))
    tab_res *= -1 
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*6+row.date.minute//10] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/35: dict_res
247/36: dict_res['LOT2-BERMUDE']
247/37: plt.imshow(dict_res['LOT2-BERMUDE'], aspect=Auto, interpolation=None);plt.show()
247/38: plt.imshow(dict_res['LOT2-BERMUDE'], aspect=auto, interpolation=None);plt.show()
247/39: plt.imshow(dict_res['LOT2-BERMUDE'], interpolation=None);plt.show()
247/40:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1 
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/41: plt.imshow(dict_res['LOT2-BERMUDE'], interpolation=None);plt.colorbar(); plt.show()
247/42: plt.imshow(dict_res['LOT2-BERMUDE'], interpolation=None);plt.colorbar(); plt.show()
247/43: plt.imshow(dict_res['LOT2-BERMUDE'], interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/44: plt.imshow(dict_res['LOT2-BERMUDE'].T, interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/45: plt.imshow(np.log10(1+dict_res['LOT2-BERMUDE'].T,) interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/46: plt.imshow(np.log10(1+dict_res['LOT2-BERMUDE'].T), interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/47: plt.imshow(np.log10(2+dict_res['LOT2-BERMUDE'].T), interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/48:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*6))
    tab_res *= -1 
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*6+row.date.minute//10] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/49: plt.imshow(np.log10(2+dict_res['LOT2-BERMUDE'].T), interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/50:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= -1 
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/51: plt.imshow(np.log10(2+dict_res['LOT2-BERMUDE'].T), interpolation=None, aspect='auto');plt.colorbar(); plt.show()
247/52: plt.imshow(dict_res['BERMUDE'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BERMUDE');plt.show()
247/53: plt.imshow(dict_res['LOT2-BERMUDE'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BERMUDE');plt.show()
247/54: plt.imshow(np.log10(dict_res['LOT2-BERMUDE'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BERMUDE');plt.show()
247/55:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= 0
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/56: plt.imshow(np.log10(dict_res['LOT2-BERMUDE'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BERMUDE');plt.show()
247/57: %history -f NEW_CAL.txt
247/58: df = pd.read_pickle("results_all_JAM.pkl")
247/59: df
247/60: df = df.dropna()
247/61: df
247/62:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= 0
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/63: df['lot'] = df['file'].str.split('/', expand=True)[0]
247/64: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
247/65: df
247/66: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
247/67: df
247/68:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= 0
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
247/69: plt.imshow(np.log10(dict_res['LOT2-BERMUDE'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BERMUDE');plt.show()
247/70: plt.imshow(np.log10(dict_res['LOT2-GUYANNE'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('GUYANNE');plt.show()
247/71: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
247/72: df.sort_values('nb_positif').iloc[-15:-25].nb_positif
247/73: df.sort_values('nb_positif').iloc[-15:-25]
247/74: df.sort_values('nb_positif').iloc[-25:-15]
247/75: df.sort_values('nb_positif').iloc[-25:-15].file
247/76: tmp = df.sort_values('nb_positif').iloc[-25:-15].file
247/77: tmp.to_csv("detec_JAM.csv")
247/78: df.sort_values('nb_positif').iloc[-50:-40].nb_positif
247/79: df.sort_values('nb_positif').iloc[-25:-15].nb_positif
247/80: df.sort_values('nb_positif').iloc[-50:-40].nb_positif
247/81: tmp = df.sort_values('nb_positif').iloc[-50:-40].file
247/82: tmp.to_csv("detec_JAM_50_40.csv")
253/1: import numpy as np
253/2: import pandas as pd
253/3: import matplotlib.pyplot as plt
253/4: df = pd.read_pickle("results_all_BON.pkl")
253/5: df = df.dropna()
253/6: df['lot'] = df['file'].str.split('/', expand=True)[0]
253/7: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
253/8: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
253/9: df
253/10:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= 0
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
253/11: plt.imshow(np.log10(dict_res['LOT2-BON'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BON');plt.show()
253/12: nb_day,
253/13: df['date'].years
253/14: df['date'].year
253/15: df['date'].years
253/16: df['date'].iloc[0]
253/17: df['date'].iloc[0].years
253/18: df['date'].iloc[0].year
253/19: df['file'].iloc[0]
253/20: df['file'] = df['file'].str.replace("/2122", "/2021")
253/21: df['file']
253/22: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
253/23:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= 0
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
253/24: nb_day
253/25: plt.imshow(np.log10(dict_res['LOT2-BON'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('BON');plt.show()
253/26: df.sort_values('nb_positif').iloc[-50:-10].nb_positif
253/27: tmp = df.sort_values('nb_positif').iloc[-50:-10].file
253/28: tmp.to_csv("detec_BON_50_10.csv")
254/1: %run compute_quantile.py
254/2: cd result_both/
254/3: %run compute_quantile.py
254/4: df = pd.read_pickle("results_all_PNN.pkl")
254/5: df
254/6: df = df.dropna()
254/7: df
254/8: df.sort_values('nb_positif')
254/9: df.sort_values('nb_positif').nb_positif
254/10: %run compute_quantile.py
254/11: df = pd.read_pickle("results_all_PNN.pkl")
254/12: df = df.dropna()
254/13: df
254/14: df.sort_values('nb_positif').nb_positif
254/15: df.sort_values('nb_positif').file
254/16: tmp = df.sort_values('nb_positif')
254/17: tmp.to_csv("detec_PNN.csv")
255/1: 1024*2
255/2: 1024*2+1
256/1: %run compute_quantile.py
257/1: %run compute_quantile.py
257/2: ls
257/3: %run compute_quantile.py
254/18: exi
258/1: %run compute_quantile.py
259/1: %run compute_quantile.py
259/2: df = pd.read_pickle("results_all_PNN.pkl")
259/3: df = df.dropna()
259/4: df
259/5: df.sort_values('nb_positif').file
259/6: df.sort_values('nb_positif').nb_positif
259/7: df.sort_values('nb_positif').nb_positif
259/8: tmp = df.sort_values('nb_positif')
259/9: tmp
259/10: tmp
259/11: tmp = tmp.drop(["q80", "q90", "q95", "q99", "prediction", "hist"])
259/12: tmp = tmp.drop(columns=["q80", "q90", "q95", "q99", "prediction", "hist"])
259/13: tmp
259/14: tmp.to_csv("detec_PNN.csv")
259/15: df
259/16: df.sort_values('nb_positif').hist
259/17: df.file = df.file.str.replace("PORTNOUVELLE_FINALE/", "")
259/18: df.file
259/19: df
259/20: df.file = df.file.str.replace("_pred.npz", "")
259/21: df.file
259/22: df
259/23: df = df.drop( columns="hist")
259/24: df
259/25: df.drop(columns="prediction").to_csv("res_PNN.csv")
259/26: df.drop(columns=["q80", "q90", "q95", "q99", "nb_positif"]).to_csv("pred_PNN.csv")
272/1: import matplotlib.pyplot as plt
272/2: import numpy as np
272/3: import scipy as scp
272/4: import pandas as pd
272/5: ls
272/6: %run compute_quantile.py
272/7: %run compute_quantile.py
272/8: df = pd.read_pickle("results_all_LOT1.pkl")
272/9: df = df.dropna()
272/10: df['lot'] = df['file'].str.split('/', expand=True)[0]
272/11: df
272/12: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_')[0]
272/13: df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
272/14: df['sess'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
272/15: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_', expand=True)[0]
272/16: df['file']
272/17: df['file']
272/18: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
272/19: df['date']
272/20: df['date'].max()
272/21: df['date'].min()
272/22: df['sess'].min()
272/23: df['sess'].max()
272/24: df
272/25: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
272/26: df['sess']
272/27: df['file'].str.split('/', expand=True)[1].str.split('[0-9]_', expand=True)[0]
272/28: df['file'].str.split('[A-Z]_', expand=True)[1].str.split('[0-9]_', expand=True)[0]
272/29: df['file'].str.split('[A-Z]_', expand=True)[1]
272/30: df['file'].str.split('/', expand=True)[1].str.split('[0-9]_', expand=True)[0]
272/31: df['file'].str.split('/', expand=True)[1].str.split('[0-9]⁴_', expand=True)[0]
272/32: df['file'].str.split('/', expand=True)[1].str.split('[0-9]4_', expand=True)[0]
272/33: df['file']
272/34: df['file'].str.split('/', expand=True)[1].str.split('[0-9]{4}_', expand=True)[0]
272/35: df['file'].str.split('/', expand=True)[1].str.split('[0-9]{4}_', expand=True)[0]
272/36: df['file'].str.split('/*[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}_', expand=True)[0]
272/37: df['date'].years = df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}_', expand=True)[0]
272/38: df['date'].years
272/39: df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}_', expand=True)[0]
272/40: df['file'].str.split('_[0-9]{8}_', expand=True)
272/41: df['file'].str.split('_[0-9]{8}_', expand=True)[1]
272/42: df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}/', expand=True)[0]
272/43: df['file']= df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}/', expand=True)[0]
272/44: df['file']
272/45: df['date'].years > 2022 ? df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}/', expand=True)[0]
272/46: df['date'].years > 2022 ? df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}/', expand=True)[0] : 0
272/47: df['date'].years > 2022
272/48: ls
272/49: %run compute_quantile.py
272/50: %run compute_quantile.py
272/51: import glob
272/52: import glob
272/53: %run compute_quantile.py
272/54: df = pd.read_pickle("results_all_CCS.pkl")
272/55: df
272/56: df.sort_values('nb_positif').
272/57: df.sort_values('nb_positif')
272/58: df
272/59: df.hist
272/60: df.hist
272/61: df.hist.hist
272/62: df.iloc[0]
272/63: df.iloc[0].hist
272/64: df.iloc[0]
272/65: df.iloc[0]['hist']
272/66: df.['hist'].sum()
272/67: df['hist'].sum()
272/68: plt.plot(df['hist'].sum()); plt.show()
272/69: df.sort_values('nb_positif')
272/70: df.sort_values('nb_positif')
272/71: plt.plot(df['nb_positif']);plt.show()
272/72: df.sort_values('nb_positif')[-10:]
272/73: df.sort_values('nb_positif')[-10:].file
272/74: tmp = df.sort_values('nb_positif')[-10:].file
272/75: tmp.to_csv("detec_CCS.csv")
272/76: df.sort_values('nb_positif')[-10:].nb_positif
272/77: df.sort_values('nb_positif')[-1]
272/78: df.sort_values('nb_positif')[-1:]
272/79: df.sort_values('nb_positif').iloc[-1]
272/80: tmp = df.sort_values('nb_positif').iloc[-1]
272/81: tmp
272/82: plt.plot(tmp.prediction);plt.show()
272/83: plt.plot(tmp.prediction);plt.show()
272/84: tmp = df.sort_values('nb_positif').iloc[-100]
272/85: tmp
272/86: tmp = df.sort_values('nb_positif').iloc[-200]
272/87: tmp
272/88: tmp/file
272/89: tmp.file
272/90: %run compute_quantile.py
272/91: %run compute_quantile.py
272/92: df = pd.read_pickle("results_all_CCS.pkl")
272/93: df
272/94: %run compute_quantile.py
272/95: df = pd.read_pickle("results_all_CCS.pkl")
272/96: df
272/97: df.sort_values('nb_positif')
272/98: tmp = df.sort_values('nb_positif')[-10:].file
272/99: tmp.to_csv("detec_CCS.csv")
272/100: tmp = df.sort_values('nb_positif').iloc[-10]
272/101: tmp
272/102: plt.plot(tmp.prediction);plt.show()
273/1: %run compute_quantile.py
273/2: %run compute_quantile.py
273/3: %run compute_quantile.py
273/4: df = pd.read_pickle("results_all_LOT2.pkl")
273/5: df
273/6: df = df.dropna()
273/7: df['lot'] = df['file'].str.split('/', expand=True)[0]
273/8: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
273/9: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
273/10: df
273/11:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/12: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/13:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*3))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*3+row.date.minute//20] += 1#row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/14: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/15:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1#row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/16: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/17:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*10))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*10+row.date.minute//6] += 1#row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/18: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/19:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*10))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*10+row.date.minute//6] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/20: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/21:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/22: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/23: df.sort_values('date')
273/24: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/25: df.sort_values('date')
273/26: plt.imshow(dict_res['LOT2-JAM'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/27: df.sort_values('date')
273/28: df.sort_values('date')[0]
273/29: df.sort_values('date').iloc[0]
273/30: df.sort_values('date').iloc[100]
273/31: df.sort_values('date').iloc[120]
273/32: df.sort_values('date').iloc[110]
273/33: df.sort_values('date')[100:110]
273/34: df.sort_values('date')[100:120]
273/35: df.sort_values('date')[100:120].nb_positif.max()
273/36: plt.imshow(dict_res['LOT2-JAM'], origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/37: plt.imshow(dict_res['LOT2-JAM'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/38: df.sort_values('date').iloc[110]
273/39: df.sort_values('date').iloc[120]
273/40: df.sort_values('date').iloc[125]
273/41: df
273/42: df.sort_values('date').iloc[125]
273/43: df = df.sort_values('file')
273/44: df
273/45:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*10))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*10+row.date.minute//6] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/46:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*5))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*5+row.date.minute//12] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/47: plt.imshow(dict_res['LOT2-JAM'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/48:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/49: plt.imshow(dict_res['LOT2-JAM'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/50: df
273/51: df = df.sort_values('file')
273/52: df
273/53: plt.imshow(dict_res['LOT2-JAM'].T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/54: plt.imshow(np.log10(dict_res['LOT2-JAM']).T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/55: plt.imshow(np.log10(dict_res['LOT2-JAM']).T, aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/56: plt.imshow(np.log10(dict_res['LOT2-JAM']).T, origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/57: plt.imshow(np.log10(dict_res['LOT2-JAM']).T, aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/58: plt.imshow(np.fliptd(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/59: plt.imshow(np.flipud(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/60:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24*5))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour*5+row.date.minute//12] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/61: plt.imshow(np.flipud(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/62:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/63: plt.imshow(np.flipud(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/64: df.sort_values('date').iloc[100]
273/65: df.sort_values('date').iloc[0]
273/66: df.sort_values('date').iloc[180]
273/67: df.sort_values('date').iloc[250]
273/68: df.sort_values('date').iloc[300]
273/69: df.sort_values('date').iloc[350].
273/70: df.sort_values('date').iloc[350]
273/71: df.sort_values('date').iloc[400]
273/72: df.sort_values('date').iloc[360]
273/73: df.sort_values('date').iloc[300]
273/74: df.sort_values('date').iloc[310]
273/75: df.sort_values('date').iloc[320]
273/76: df.sort_values('date').iloc[330]
273/77: df.sort_values('date')[300:350].max()
273/78: df.sort_values('date')[300:350].nb_prediction.max()
273/79: df.sort_values('date')[300:350].nb_positif.max()
273/80: df.sort_values('date').iloc[300]
273/81: df.sort_values('date').iloc[330]
273/82: df.sort_values('date').iloc[340]
273/83: df.sort_values('date').iloc[350]
273/84: plt.imshow(np.flipud(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/85: plt.imshow(np.flipud(np.log10(dict_res['LOT2-JAM']).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/86: df
273/87:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    break
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/88: grp
273/89:  (grp['date'].max())
273/90:  (grp['date'].min())
273/91:     ...:     nb_day = (grp['date'].max() - grp['date'].min()).days + 1
273/92:      nb_day = (grp['date'].max() - grp['date'].min()).days + 1
273/93: tab_res = np.ones((nb_day, 24))
273/94:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        break
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/95: row
273/96: (row.date - grp['date'].min()).days
273/97: row.date.hour
273/98: plt.imshow(np.fliplr(np.flipud(np.log10(dict_res['LOT2-JAM'])).T), origin='lower', aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/99: plt.imshow(np.log10(dict_res['LOT2-JAM'].T),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/100: tab_res[(row.date - grp['date'].min()).days, row.date.hour]= -1000
273/101: plt.imshow(np.log10(dict_res['LOT2-JAM'].T),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/102: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/103:  row.date.hour
273/104:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/105: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/106: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/107:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        break
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/108: [(row.date - grp['date'].min()).days, row.date.hour]
273/109: tab_res[(row.date - grp['date'].min()).days, row.date.hour]= -1000
273/110: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/111: tab_res[(row.date - grp['date'].min()).days, row.date.hour+1]= +1000
273/112: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/113:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
    break
273/114: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/115:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/116:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/117: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/118:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24))
    tab_res *= -1
    for idx, row in grp.iterrows():
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += True #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/119: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/120:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += idx #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/121: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/122:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(raw.date)
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += idx #row['nb_positif']
        break
    dict_res['-'.join(s)] = tab_res
273/123:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(row.date)
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += idx #row['nb_positif']
        break
    dict_res['-'.join(s)] = tab_res
273/124: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/125: idx
273/126:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(row.date)
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1  #row['nb_positif']
        break
    dict_res['-'.join(s)] = tab_res
273/127: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/128:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(row.date)
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1  #row['nb_positif']
        plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
    dict_res['-'.join(s)] = tab_res
273/129:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(row.date)
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1  #row['nb_positif']
        plt.imshow(tab_res.T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
    dict_res['-'.join(s)] = tab_res
273/130: df
273/131:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        print(row.date)
        break
        print([(row.date - grp['date'].min()).days, row.date.hour])
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1  #row['nb_positif']
        plt.imshow(tab_res.T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
    dict_res['-'.join(s)] = tab_res
273/132: row.date
273/133: grp['date'].min()
273/134: row.date.days
273/135: row.date
273/136: ( grp['date'].min()).days
273/137: grp['date'].min()
273/138: grp['date'].min().hour
273/139: grp['date'].min().day
273/140: row.date.day
273/141: row.date.day
273/142: row.date
273/143:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[row.date.day - grp['date'].min().day, row.date.hour] += 1  #row['nb_positif']
        plt.imshow(tab_res.T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
    dict_res['-'.join(s)] = tab_res
273/144:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[row.date.day - grp['date'].min().day, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/145: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/146: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/147:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min()).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/148: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/149:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor.day).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/150: row
273/151: row.date
273/152: row.date.floor.hour
273/153: row.date.floor.hours
273/154: row.date.floor('hours')
273/155: row.date.floor(freq='H')
273/156: row.date.floor(freq='D')
273/157: row.date.floor.D
273/158: row.date.floor('D')
273/159:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 1
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/160: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/161:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min()).days + 10
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/162: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/163:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D').days) + 1    
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/164:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour] += 1  #row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/165: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/166:
dict_res = {} 
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/167: plt.imshow(dict_res['LOT2-JAM'].T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/168: plt.imshow(plt.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/169: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show()
273/170: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.show(block=False)
273/171: df.iloc[300].date
273/172: df.iloc[600].date
273/173: df.iloc[400].date
273/174: df.iloc[350].date
273/175: df.iloc[360].date
273/176: df.iloc[360]
273/177: df.iloc[360].date
273/178: df.iloc[380].date
273/179: df.iloc[370].date
273/180: df.[360:370].nb_positif
273/181: df.[360:370].nb_positif
273/182: df[360:370].nb_positif
273/183: df[360:370].nb_positif.max()
273/184: df[365:370].date
273/185: df[360:370].date
273/186: df[360:370]
273/187: df
273/188: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM'); ;plt.show(block=False)
273/189: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xtick(np.aragne(nb_days), np.arange(nb_days)*12);plt.show()
273/190: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xticks(np.aragne(nb_days), np.arange(nb_days)*12);plt.show()
273/191: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xticks(np.arange(nb_days), np.arange(nb_days)*12);plt.show()
273/192: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xticks(np.arange(nb_day), np.arange(nb_day)*12);plt.show()
273/193: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xticks(np.arange(nb_day), np.arange(nb_day)*12);plt.show()
273/194: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.xticks(np.arange(nb_day), np.arange(nb_day)*10);plt.show()
273/195: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(24*2), np.arange(24));plt.show()
273/196: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(24), np.arange(24*2));plt.show()
273/197: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 2), np.arange(12));plt.show()
273/198: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24));plt.show()
273/199: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xtick(np.arange(0,nb_day, 10),['A','B','C']);plt.show()
273/200: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(0,nb_day, 10),['A','B','C']);plt.show()
273/201: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(0,nb_day, 10),['A','B','C', 'D']);plt.show()
273/202: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(0,nb_day, 10),['A','B','C', 'D']);plt.show()
273/203: grp
273/204: grp['date'].min().floor('D')
273/205: grp['date'].min().floor('D')
273/206: pd.date_range(grp['date'].min(), grp['date'].max())
273/207: pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')
273/208: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/209: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/210: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/211: pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')
273/212: print(pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'))
273/213: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')); plt.;plt.show()
273/214: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')); plt.;plt.show()
273/215: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/216: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/217: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=45 );plt.show()
273/218: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/219: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=60);plt.show()
273/220: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/221: pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')
273/222: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/223: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/224: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/225: pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')
273/226: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/227: import matplotlib.dates as mdates
273/228: xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/229: plt.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/230: plt.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/231: plt.gca(set_major_formatter(mdates.DateFormatter('%Y-%m-%D')))
273/232: plt.gca().set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/233: plt.gca().axis.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/234: plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/235: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/236: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/237: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));
273/238: plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%D'))
273/239: plt.show()
273/240: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));
273/241: plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
273/242: plt.show()
273/243: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));
273/244: plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%D'))
273/245: plt.show()
273/246: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));
273/247: plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
273/248: plt.show()
273/249: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'));plt.show()
273/250: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotate=90);plt.show()
273/251: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1), pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')), rotate=90);plt.show()
273/252: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/253: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/254: tmp = grp['date'].max()).floor('D')
273/255: tmp =pd.date_range(grp['date'].min(), grp['date'].max()).floor('D')
273/256: tmp
273/257: tmp.str()
273/258: tmp
273/259: tmp.to_pydatetime()
273/260: datetime.strptime(tmp, '%Y-%m-%d')
273/261: tmp
273/262: str(tmp)
273/263: rmp
273/264: tmp
273/265: tmp[0].strptime
273/266: tmp[0].strptime()
273/267: tmp[0].strptime('%Y-%m-%d')
273/268: tmp[0].strptime(tmp[0], '%Y-%m-%d')
273/269: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+1),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/270: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+2),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/271:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/272: plt.imshow(np.log10(dict_res['LOT2-JAM'].T+2),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/273: %history -f NEW_CAL2.txt
273/274: %run compute_quantile.py
273/275: df
273/276:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
    dict_res['-'.join(s)] = tab_res
273/277:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), type=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/278:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div))
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/279: plt.imshow(np.log10(dict_file['LOT2-JAM']),  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/280: plt.imshow(np.log10(dict_file['LOT2-JAM']).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/281:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/282: plt.imshow(np.log10(dict_file['LOT2-JAM']).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/283:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.one((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/284:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.ones((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/285: plt.imshow(np.log10(dict_file['LOT2-JAM']).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/286: tab_file
273/287:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/288: tab_file
273/289: plt.imshow(np.log10(dict_file['LOT2-JAM']).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/290: plt.imshow((dict_file['LOT2-JAM']).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title('JAM');plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/291: %run compute_quantile.py
273/292: %history -f NEW_CAL3.txt
273/293: df = pd.read_pickle("results_all_LOT2.pkl")
273/294: df
273/295: df = df.dropna()
273/296: df = df.sort_values('file')
273/297: df['lot'] = df['file'].str.split('/', expand=True)[0]
273/298: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
273/299: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
273/300: df
273/301:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = tab_res
    dict_file['-'.join(s)] = tab_file
273/302: dict_res.keys()
273/303:
for key in dict_res.keys():
    plt.imshow(np.log10(dict_file[key]).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(grp['date'].min(), grp['date'].max()).floor('D'), rotation=90);plt.show()
273/304: plt.show()
273/305: plt.close()
273/306:
dict_res = {}
dict_file = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
273/307:
for key in dict_res.keys():
    break
273/308: key
273/309: tab_res, tab_file, datemin,  datemax = dict_res[key]
273/310: tab_res.shape
273/311: datemin
273/312: datemax
273/313: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/314: key
273/315: key = 'LOT2-JAM'
273/316: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/317: tab_res, tab_file, datemin,  datemax = dict_res[key]
273/318: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/319: datemin,
273/320: datemax
273/321: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.show()
273/322: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.show()
273/323: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.show()
273/324: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(nb_day-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/325: pd.date_range(datemin, datemax).floor('D')
273/326: nb_day-1
273/327: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange(datemax-datemin-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/328: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days-1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/329: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/330: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/331: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/332: %history -f NEW_CAL4.txt
273/333:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
tab_file

for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/334: plt.close()
273/335:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.savefig("%s.pdf"%key)
273/336: keu
273/337: key
273/338: tab_res, tab_file, datemin,  datemax = dict_res[key]
273/339: datemin,
273/340: datemax
273/341: key =  dict_res.keys()
273/342: tab_res, tab_file, datemin,  datemax = dict_res[key]
273/343: dict_res
273/344: key
273/345: tab_res, tab_file, datemin,  datemax = dict_res['LOT2-ANGUILLA']
273/346: tab_res
273/347: datemin
273/348: datemax
273/349: df
273/350: tmp = df['sess' == 'BERMUDE']
273/351: tmp = df[df['sess'] == 'BERMUDE']
273/352: tmp
273/353: tmp.nb_positif.max()
273/354: tmp.nb_positif.min()
273/355: tab_res
273/356: tab_res.shape
273/357: key
273/358:
for key in dict_res.keys():
    key = 'LOT2-BERMUDE'
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show(block=False)%;plt.savefig("%s.pdf"%key)
273/359:
for key in dict_res.keys():
    key = 'LOT2-BERMUDE'
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show(block=False)for key in dict_res.keys():
    key = 'LOT2-BERMUDE'
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show(block=False)#;plt.savefig("%s.pdf"%key)
273/360:
for key in dict_res.keys():
    key = 'LOT2-BERMUDE'
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show(block=False)#;plt.savefig("%s.pdf"%key)
273/361: df
273/362: tmp
273/363: tmp.iloc[100].date
273/364: tmp.iloc[1000].date
273/365: tmp.iloc[1800].date
273/366: tmp.iloc[2000].date
273/367: tmp.iloc[2500].date
273/368: tmp.iloc[2700].date
273/369: tmp.iloc[2700].nb_positif
273/370: tmp.iloc[2701].nb_positif
273/371: tmp.iloc[2701].date
273/372: tmp.iloc[2702].nb_positif
273/373: tmp.iloc[5000].date
273/374: tmp.iloc[6000].date
273/375: tmp.iloc[6000].nb_positif
273/376: tmp.iloc[6000].date
273/377: tmp.iloc[10000].date
273/378: tmp.iloc[15000].date
273/379: tmp.iloc[13000].date
273/380: tmp.iloc[14000].date
273/381: tmp.iloc[13500].date
273/382: tmp.iloc[13300].date
273/383: tmp.iloc[13100].date
273/384: tmp.iloc[13100].date
273/385: tmp.iloc[13200].date
273/386: tmp.iloc[13202].date
273/387: tmp.iloc[13202].nb_positif
273/388: tmp.iloc[13302].date
273/389: tmp.iloc[13402].date
273/390: tmp.iloc[13452].date
273/391: tmp.iloc[13445].date
273/392: tmp.iloc[13442].date
273/393: tmp.iloc[13442].nb_positif
273/394: tmp.iloc[13442].date
273/395: tmp.iloc[13642].date
273/396: tmp.iloc[13672].date
273/397: tmp.iloc[13678].date
273/398: tmp.iloc[13682].date
273/399: tmp.iloc[13682].nb_positif
273/400: key
273/401: key = 'LOT2-BON'
273/402: tab_res, tab_file, datemin,  datemax = dict_res['LOT2-BON']
273/403: tab_res
273/404: datemin
273/405: datemax
273/406: plt.imshow(np.log10(tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, 1), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.show()
273/407: df
273/408: df[df.date.year>2022]
273/409: df[df.date.years>2022]
273/410: [df.date>2022]
273/411: df.date>2022
273/412: df.date.year>2022
273/413: df.date.years>2022
273/414: df.date
273/415: df.date.year
273/416: df.date.years
273/417: df.date.year
273/418: df.date
273/419: df.date.day
273/420: df
273/421: tmp = df.iloc[10]
273/422: tmp
273/423: tmp.date.year
273/424: tmp.date.year>2022
273/425: tmp.date.year< 2020
273/426: df
273/427: df['file']
273/428: df
273/429: tmp = df.copy()
273/430: tmp
273/431: tmp.replace(year=2021)
273/432: tmp.replace(years=2021)
273/433: tmp
273/434:
for elem in tmp.iterrows:
    break
273/435:
for elem in tmp.iterrows():
    break
273/436: elem
273/437: elem.date.years
273/438: elem.date
273/439: elem
273/440:
for idx, elem in tmp.iterrows():
    break
273/441: elem
273/442: elem.date.years
273/443: elem.date.year
273/444: elem.date.year < 2021
273/445: elem.date.year = elem.date.year == 2121 ? 2021 or elem.date.year
273/446: elem.date.year = (elem.date.year == 2121 ? 2021 or elem.date.year)
273/447: elem.date.year = (elem.date.year == 2121 ? 2021 : elem.date.year)
273/448: elem.date.year = elem.date.year == 2121 ? 2021 : elem.date.year
273/449: elem.date.year == 2121 ? 2021 : elem.date.year
273/450: elem.date.year = 2021 if elem.date.year == 2121 else elem.date.year
273/451:  2021 if elem.date.year == 2121 else elem.date.year
273/452: tmp['file']
273/453: tmp['file'].str.replace('/2121', '/2021')
273/454: tmp = tmp['file'].str.replace('/2121', '/2021')
273/455: tmp = df.copy()
273/456: tmp['file'] = tmp['file'].str.replace('/2121', '/2021')
273/457: tmp['file'] = tmp['file'].str.replace('/2131', '/2021')
273/458: tmp
273/459: df['file'] = df['file'].str.replace('/2121', '/2021')
273/460: df['file'] = df['file'].str.replace('/2131', '/2021')
273/461: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
273/462: df['date'].max()
273/463: df['file'] = df['file'].str.replace('/2122', '/2022')
273/464: df['date'].max()
273/465: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
273/466: df['date'].max()
273/467: df[df['date'].max()]
273/468: df[df['date'].argmax()]
273/469: df['date'].argmax()
273/470: df['date'].max()
273/471: df[ df['date'] == df['date'].max()]
273/472: df['file'] = df['file'].str.replace('/2035', '/2021')
273/473: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
273/474: df['date'].max()
291/1: import pandas as pd
291/2: imprt
291/3: import numpy as np
291/4: import matplotlib.pyplot as plt
291/5: df = pd.read_pickle("results_all_LOT2.pkl")
291/6: cd result_both/
291/7: df = pd.read_pickle("results_all_LOT2.pkl")
291/8: df.date>2022
291/9: df['file'] = df['file'].str.replace('/2035', '/2021')
291/10: df['file'] = df['file'].str.replace('/2122', '/2022')
291/11: df['file'] = df['file'].str.replace('/2131', '/2021')
291/12: df['file']
291/13: df = df.dropna()
291/14: df = df.sort_values('file')
291/15:
df['lot'] = df['file'].str.split('/', expand=True)[0]
df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
291/16: df['date'].max()
291/17: df['file'] = df['file'].str.replace('/2121', '/2021')
291/18:
df = df.dropna()
df = df.sort_values('file')
291/19:

df['lot'] = df['file'].str.split('/', expand=True)[0]
df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
291/20: df['date'].max()
291/21:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
291/22:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.savefig("%s.pdf"%key)
291/23:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);fig.tight_layout();plt.savefig("%s.pdf"%key)
291/24:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.tight_layout();plt.savefig("%s.pdf"%key)
291/25: key
291/26:
dict_res = {}
sub_div = 6
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
291/27:
dict_res = {}
sub_div = 3
for s, grp in df.groupby([df['lot'], df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
291/28:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.tight_layout();plt.savefig("%s.pdf"%key)
291/29:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24*sub_div, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.tight_layout();plt.savefig("%s.pdf"%key)
291/30: %history -f NEW_CAL5.txt
295/1: import numpy as np
295/2: import matplotlib.pyplot as plt
295/3: import pandas as pd
295/4: import UMAP
295/5: import umpa
295/6: import umap
295/7: import umap
295/8: df = pd.read_pickle("results_all_PNN.pkl")
295/9: df = pd.read_pickle("results_all_LOT2.pkl")
295/10: df = pd.read_pickle("../results_all_LOT2.pkl")
295/11: pwd
295/12: df = pd.read_pickle("result_both/results_all_LOT2.pkl")
295/13: df
295/14: %run compute_bigdf.py
295/15: cd result_both/
295/16: %run compute_bigdf.py
295/17: %run compute_bigdf.py
295/18: %run compute_bigdf.py
295/19: %run compute_bigdf.py
295/20: %run compute_bigdf.py
296/1: %run compute_bigdf.py
297/1: %run compute_bigdf.py
297/2: %run compute_bigdf.py
297/3: %run compute_bigdf.py
297/4: %run compute_bigdf.py
297/5: %run compute_bigdf.py
297/6: %run compute_bigdf.py
297/7: %run compute_bigdf.py
297/8: %run compute_bigdf.py
297/9: %run compute_bigdf.py
297/10: %run compute_bigdf.py
300/1: import umap
300/2: import numpy as np
300/3: import matplotlib.pyplot as plt
300/4: df = pd.read_pickle("result_both/emb_all_LOT2.pkl")
300/5: import pandas as pd
300/6: df = pd.read_pickle("result_both/emb_all_LOT2.pkl")
300/7: pwd
300/8: df = pd.read_pickle("emb_all_LOT2.pkl")
300/9:     reducer = UMAP(verbose=10)
300/10: from umpa import UMAP
300/11: from umap import UMAP
300/12: df
300/13: df['lot'] = df['file'].str.split('/', expand=True)[0]
300/14: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
300/15: df
300/16: df[df.sess == 'JAM']
300/17: df[df.sess == 'JAM'][::10]
300/18: map = reducer.fit_transform(df[df.sess == 'JAM'][::10])
300/19:     reducer = UMAP(verbose=10)
300/20: map = reducer.fit_transform(df[df.sess == 'JAM'][::10])
300/21: map = reducer.fit_transform(df[df.sess == 'JAM'][::10].emb)
300/22: df[df.sess == 'JAM'][::10].emb
300/23: map = reducer.fit_transform(np.array(df[df.sess == 'JAM'][::10].emb))
300/24: np.array(df[df.sess == 'JAM'][::10].emb)
300/25: np.array(df[df.sess == 'JAM'][::10])
300/26: np.array(df[df.sess == 'JAM'][::10].emb)
300/27: map = reducer.fit_transform(np.array(df[df.sess == 'JAM'][::10].emb))
300/28: (np.array(df[df.sess == 'JAM'][::1000].emb))
300/29: map = reducer.fit_transform(df[::1000].emb)
300/30: df.iloc[0].emb
300/31: df.iloc[1].emb
300/32: tmp = np.array(df[df.sess == 'JAM'][::10].emb)
300/33: tmp.shape
300/34: tmp
300/35: tmp.vstack(tmp)
300/36: np.vstack(tmp).shape
300/37: map = reducer.fit_transform(np.vstack(df[df.sess == 'JAM'][::10].emb))
300/38:     plt.scatter(map[:, 0], map[:, 1], 3, c=df[df.sess == 'JAM'][::10].pred)
300/39: plt.show()
300/40:     plt.scatter(map[:, 0], map[:, 1], 3, c=df[df.sess == 'JAM'][::10].pred, cmap='jet'); plt.show()
300/41: plt.show()    reducer = UMAP(verbose=10)
300/42: reducer = UMAP(verbose=10)
300/43: map = reducer.fit_transform(np.vstack(df[::1000].emb))
300/44: plt.scatter(map[:, 0], map[:, 1], 3, c=df[::1000].pos, cmap='jet')
300/45: plt.show()
300/46:     reducer = UMAP(verbose=10)
300/47: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/48: plt.scatter(map[:, 0], map[:, 1], 3, c=df[::100].pos, cmap='jet')
300/49: plt.show()
300/50: reducer = UMAP(verbose=10)
300/51: map = reducer.fit_transform(np.vstack(df[df.sess == 'JAM'][::10].emb))
300/52: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[df.sess == 'JAM'][::10].pred), cmap='jet')
300/53: plt.show()
300/54:     reducer = UMAP(verbose=10)
300/55: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/56: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[::100].pred), cmap='jet')
300/57: plt.show()
300/58: map = reducer.fit_transform(np.vstack(df[df.sess == 'BERMUDE'][::10].emb))
300/59: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[df.sess == 'BERMUDE'][::10].pred), cmap='jet')
300/60: plt.close()
300/61: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[df.sess == 'BERMUDE'][::10].pred), cmap='jet')
300/62: plt.show()
300/63: map = reducer.fit_transform(np.vstack(df[df.sess == 'JAM'][::10].emb))
300/64: plt.close()
300/65: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[df.sess == 'JAM'][::10].pred), cmap='jet')
300/66: plt.show()
300/67: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/68: df
300/69: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[df.sess == 'JAM'][::10].pred), cmap='jet')
300/70: plt.close()
300/71: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[::100].sess), cmap='jet')
300/72:     reducer = UMAP(verbose=10)
300/73: map = reducer.fit_transform(np.vstack(df[np.logicial_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].emb))
300/74: map = reducer.fit_transform(np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].emb))
300/75: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].pred), cmap='jet')
300/76: plt.show()
300/77: plt.close()
300/78: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].pred), cmap='jet')
300/79: plt.show()
300/80: df.sess.unique()
300/81: color_sess = {df.sess.unique(), np.arange(10)}
300/82: color_sess = {df.sess.unique(), np.arrange(10)}
300/83: color_sess = {df.sess.unique(), np.range(10)}
300/84: color_sess = {df.sess.unique(), np.arange(10)}
300/85: color_sess = {df.sess.unique(), [0,1,2,3,4,5,6,7,8,9]}
300/86: color_sess = {list(df.sess.unique()), [0,1,2,3,4,5,6,7,8,9]}
300/87: color_sess = zip(list(df.sess.unique()), np.arrange(10))
300/88: color_sess = zip(list(df.sess.unique()), np.arange(10))
300/89: plt.close()
300/90: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/91:     reducer = UMAP(verbose=10)
300/92: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/93: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(color_sess[df[::100].sess][1]), cmap='jet')
300/94: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(color_sess[df[::100].sess]), cmap='jet')
300/95: color_sess
300/96: iter[color_sess]
300/97: next[color_sess]
300/98: next(color_sess)
300/99: tmp = dict(color_sess)
300/100: tmp
300/101: color_sess = dict(color_sess)
300/102: plt.scatter(map[:, 0], map[:, 1], 3, c=np.vstack(color_sess[df[::100].sess]), cmap='jet')
300/103: color_sess
300/104: color_sess = zip(list(df.sess.unique()), np.arange(10))
300/105: color_sess = dict(color_sess)
300/106: color_sess
300/107: color_sess['JAM']
300/108: plt.close()
300/109: plt.scatter(map[:, 0], map[:, 1], 3, c=color_sess[df[::100].sess], cmap='jet')
300/110: tmp = color_sess[df[::100].sess]
300/111: color_sess[df[::100].sess]
300/112: df[::100].sess
300/113: color_sess['JAM']
300/114: color_sess['GUA_SF']
300/115: plt.scatter(map[:, 0], map[:, 1], 3, c=color_sess[list(df[::100].sess)], cmap='jet')
300/116: df['color_sess'] = df.apply(lambda x : color_sess[x.sess])
300/117: tmp = df.apply(lambda x : color_sess[x.sess])
300/118: color_sess['GUA_SF']
300/119: df
300/120: df['color_sess'] = df.sess.apply(lambda x : color_sess[x])
300/121: df
300/122: df.color_sess.unique()
300/123: reducer = UMAP(verbose=10)
300/124: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/125: plt.scatter(map[:, 0], map[:, 1], 3, c=df[::100].color_sess, cmap='jet')
300/126: plt.show()
300/127: color_sess
300/128: plt.imshow(np.arange(10)[:, None]);plt.colorbar();plt.show()
300/129: plt.imshow(np.arange(10)[:, None], cmap='jet');plt.colorbar();plt.show()
300/130: reducer = UMAP(verbose=10)
300/131: map = reducer.fit_transform(np.vstack(df[::100].emb))
300/132: reducer = UMAP(verbose=10)
300/133: map = reducer.fit_transform(np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].emb))
300/134: plt.close()
300/135: plt.scatter(map[:, 0], map[:, 1], 3, c=np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].color_sess, cmap='jet')
300/136: plt.scatter(map[:, 0], map[:, 1], 3, c=df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].color_sess, cmap='jet')
300/137: plt.show()
300/138: reducer = UMAP(verbose=10)
300/139: map = reducer.fit_transform(np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'GUA_AB')][::10].emb))
300/140: plt.scatter(map[:, 0], map[:, 1], 3, c=df[np.logical_or(df.sess == 'JAM', df.sess == 'GUA_AB')][::10].color_sess, cmap='jet')
300/141: plt.show()
300/142: reducer = UMAP(verbose=10, n_components=3)
300/143: map = reducer.fit_transform(np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'GUA_AB')][::10].emb))
300/144:
plt.figure()
ax = fig.add_subplot(projection='3d')
ax.scatter(*map.T, s=3,  c=df[np.logical_or(df.sess == 'JAM', df.sess == 'GUA_AB')][::10].color_sess, cmap='jet')
plt.show()
300/145:
fig = plt.figure()
ax = fig.add_subplot(projection='3d')
ax.scatter(*map.T, s=3,  c=df[np.logical_or(df.sess == 'JAM', df.sess == 'GUA_AB')][::10].color_sess, cmap='jet')
plt.show()
300/146: reducer = UMAP(verbose=10, n_components=3)
300/147: map = reducer.fit_transform(np.vstack(df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].emb))
300/148:
fig = plt.figure()
ax = fig.add_subplot(projection='3d')
ax.scatter(*map.T, s=3,  c=df[np.logical_or(df.sess == 'JAM', df.sess == 'BERMUDE')][::10].color_sess, cmap='jet')
plt.show()
300/149: color_sess
300/150: plt.imshow(np.arange(10)[:, None], cmap='jet');plt.colorbar();plt.show()
300/151: df
300/152: (df.pred > 0.5).sum()
300/153: df
300/154: (df.pred > 0.5).sum() / len(df)
300/155: (df[df.sess='JAM'].pred > 0.5).sum()
300/156: (df[df.sess=='JAM'].pred > 0.5).sum()
300/157: df[df.sess=='JAM'].pred > 0.5).sum()
300/158: (df[df.sess=='JAM'].pred > 0.5).sum()
300/159: (df[df.sess=='JAM'].pred > 0.5).sum()
300/160: len(df[df.sess=='JAM'])
300/161: 436520/4395
300/162: 4395/436520
301/1: import matplotlib.pyplot as plt
301/2: import numpy as np
301/3: import pandas as pd
301/4: df = pd.read_pickle("emb_all_LOT9.pkl")
301/5: df = pd.read_pickle("results_all_LOT9.pkl")
301/6: df
301/7: df.sort_values('nb_positif')
301/8: df = df.dropna()
301/9: df.sort_values('nb_positif')
301/10: df.sort_values('nb_positif').iloc[-1]
301/11: df.sort_values('nb_positif')[-1:-10]
301/12: df.sort_values('nb_positif')[-10:].nb_positif
301/13: df.sort_values('nb_positif')[-10:].file
301/14: tmp = df.sort_values('nb_positif')[-10:].file
301/15: tmp = tmp.drop(columns=["q80", "q90", "q95", "q99", "prediction", "hist"])
301/16: tmp
301/17: tmp.to_csv("detec_LOT9.csv")
301/18: df.sort_values('nb_positif')[-50:-20].nb_positif
301/19: df.sort_values('nb_positif')[-80:-50].nb_positif
301/20: df.sort_values('nb_positif')[-100:-80].nb_positif
301/21: df.sort_values('nb_positif')[-140:-100].nb_positif
301/22: df.sort_values('nb_positif')[-140:-100].file
301/23: tmp = df.sort_values('nb_positif')[-140:-100].file
301/24: tmp.to_csv("detec_LOT9.csv")
375/1: import torch
375/2: torch.zeros(1)
375/3: torch.zeros(1).gpu()
400/1: import glob
400/2: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021//*/*/*/*/*.wav")
400/3: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/EXP6/*/*/*/*.wav")
400/4: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/*/*/*/*.wav")
400/5: len(tmp)
400/6: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/*/*/*/")
400/7: tmp
400/8: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/WAV/*/*/")
400/9: tmp
400/10: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/Wav/*/*/")
400/11: tmp
400/12: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/Wavs/*/*/")
400/13: tmp
400/14: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/Wavs/*")
400/15: tmp
400/16: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp6/Wavs/*/*.wav")
400/17: tmp
400/18: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp*/Wavs/*/*.wav")
400/19: tmp = glob.glob("/nfs/NAS3/SABIOD/SITE/ANTILLES_CCS_2021/Exp1/Wavs/*/*.wav")
400/20: tp
400/21: tmp
401/1: import glob
401/2: import numpy as np
401/3: import matplotlib.pyplot as plt
401/4: import pandas as pd
401/5: df = pd.read_pickle("emb_all_CCS.pkl")
401/6: df
401/7: df.sort_values(df.file)
401/8: df.sort_values(file)
401/9: df.sort_values("file")
401/10: df = df.sort_values("file")
401/11: df = df.dr
401/12: df = df.dropna()
401/13: df
401/14: df['file'].str.split('_[0-9]{8}_', expand=True)[1].str.split('[0-9]{4}/', expand=True)[0]
401/15: df['file'].str.replace("_emb.npz")
401/16: df['file'].str.replace("_emb.npz")
401/17: df['file'].str.replace("_emb.npz", "")
401/18: df = df['file'].str.replace("_emb.npz", "")
401/19: df
401/20: df
401/21: df = pd.read_pickle("emb_all_CCS.pkl")
401/22: df = df.dropna()
401/23: df = df.sort_values("file")
401/24: df
401/25: df = df['file'].str.replace("_emb.npz", "")
401/26: df
401/27: df['file']
401/28: df
401/29: df = pd.read_pickle("emb_all_CCS.pkl")
401/30: df = df.dropna()
401/31: df
401/32: df['file'] = df['file'].str.replace("_emb.npz", "")
401/33: df
401/34: df['file']
401/35: df['file'].iloc[0]
401/36: df
401/37: df['file'].str.split('/', expand=True)[1]
401/38: df['sess'] = df['file'].str.split('/', expand=True)[1]
401/39: df['file'].str.split('/', expand=True)[3]
401/40: df['file'].str.split('/', expand=True)[5]
401/41: df['file'].str.split('/', expand=True)[4]
401/42: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[4].str.split('_', expand=True)[1], format="%Y%m%d_%H%M%S")
401/43: df
401/44: df = df.sort_values("file")
401/45: df
401/46: df.iloc[1]
401/47: df.iloc[1].file
401/48: df.iloc[0].file
401/49: 482.0/60
401/50: df.iloc[2]
401/51: df.iloc[2].file
401/52: df.iloc[3].file
402/1: import pandas as pd
402/2: import numpy as np
402/3: import scipy as scp
402/4: import matplotlib.pyplot as plt
402/5: df = pd.read_pickle("results_all_CCS.pkl")
402/6: df
402/7: df = df.sort_values("file")
402/8: df
402/9: df = df.dropna()
402/10: df
402/11: 9584.0/60
403/1: import matplotlib.pyplot as plt
403/2: import numpy as np
403/3: import matplotlib.pyplot as plt
403/4: data = np.load(filename)['arr_0']
403/5: filename = "result_both/CCS/Exp6/Wavs/20210803/CCS_20210803_201046_195.wav_pred.npz"
403/6: data = np.load(filename)['arr_0']
403/7: pwd
403/8: filename = "CCS/Exp6/Wavs/20210803/CCS_20210803_201046_195.wav_pred.npz"
403/9: data = np.load(filename)['arr_0']
403/10: data
403/11: plt.plot(data);plt.show()
403/12: pos = data> 0.5
403/13: pos
403/14: pos.sum()
403/15: import glob
403/16: filename
403/17: glob.glob('CCS/Exp6/Wavs/20210803/CCS_20210803*_pred.npz')
403/18: glob.glob('CCS/Exp6/Wavs/20210803/CCS_20210803_2158*_pred.npz')
404/1: import glob
404/2: import pandas as pd
404/3: import scipy as scp
404/4: import matplotlib.pyplot as plt
404/5: df = pd.read_pickle("results_all_CCS.pkl")
404/6: df = df.sort_values("file")
404/7: df
404/8: df = df.dropna()
404/9: df
404/10: df['file'].unique()
404/11: len(df['file'].unique())
404/12: df
404/13:     files = glob.glob('CCS/*/*/*/*_emb.npz')
404/14: files
404/15: %run compute_bigdf.py
404/16: %run compute_quantile.py
404/17: df = pd.read_pickle("results_all_CCS.pkl")
404/18: df = df.dropna()
404/19: df
404/20: df = df.sort_values("file")
404/21: df
404/22: df['sess'] = df['file'].str.split('/', expand=True)[1]
404/23: df
404/24: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[4].str.split('_', expand=True)[1], format="%Y%m%d_%H%M%S")
404/25: df['file'].str.split('/', expand=True)[4].str.split('_', expand=True)[1]
404/26: df['file'].str.split('/', expand=True)[4]
404/27: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[4], format="%Y%m%d_%H%M%S")
404/28: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[4], format="CCS_%Y%m%d_%H%M%S_")
404/29: df['date']
404/30: df
404/31: df.iloc[3].file
404/32: df['file'].str.split('/', expand=True)[4].str.replace("_[0-9]{3}.wav_pred.npz", "")
404/33: df['file'].str.split('/', expand=True)[4].str.replace("CSS_[0-9]{8}_[0-9]{6}_[0-9]{3}.wav_pred.npz", "")
404/34: df['file'].str.split('/', expand=True)[4].str.replace("_[0-9]{3}.wav_pred.npz", "")
404/35: df['file'].str.split('/', expand=True)[4].str.replace("_[0-9]{3}.wav_pred.npz", "").str.replace("CCS_", "")
404/36: df['date'] = pd.to_datetime( df['file'].str.split('/', expand=True)[4].str.replace("_[0-9]{3}.wav_pred.npz", "").str.replace("CCS_", "") , format="CCS_%Y%m%d_%H%M%S_")
404/37: df['date'] = pd.to_datetime( df['file'].str.split('/', expand=True)[4].str.replace("_[0-9]{3}.wav_pred.npz", "").str.replace("CCS_", "") , format="%Y%m%d_%H%M%S")
404/38: df
404/39: df[['date', 'nb_positif']]
404/40: df[['date', 'nb_positif']].to_csv('detect_CCS.csv')
404/41: pwd
404/42: pwd
404/43: df
404/44: plt.plot(df['nb_positif'])
404/45: df
404/46: df
404/47: plt.plot(df['q90'])
404/48: plt.show()
404/49: plt.plot(np.array(df['nb_positif']))
404/50: plt.plot(np.array(df['q90']))
404/51: plt.show()
404/52: plt.plot(np.array(df['nb_positif']))
404/53: plt.plot(np.array(df['q99']))
404/54: plt.show()
404/55: df['nb_positif']
404/56: df['nb_positif'].max()
404/57: df['nb_positif'].max()/60
404/58: plt.plot(np.array(df['nb_positif'])/60)
404/59: plt.plot(np.array(df['q99']))
404/60: plt.show()
404/61: plt.plot(np.array(df['nb_positif'])/60/9.43)
404/62: plt.plot(np.array(df['q99']))
404/63: plt.show()
404/64: plt.plot(np.array(df['nb_positif'])/60/9.43)
404/65: plt.plot(np.array(df['q90']))
404/66: plt.show()
404/67: plt.plot(np.array(df['nb_positif'])/60/9.43)
404/68: plt.plot(np.array(df['q80']))
404/69: plt.show()
404/70: df[['date','file',, 'nb_positif']].to_csv('detect_CCS2.csv')
404/71: df[['date','file', 'nb_positif']].to_csv('detect_CCS2.csv')
405/1: %run compute_quantile.py
405/2:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
405/3: df = pd.read_pickle("results_all_CARIMAM.pkl")
405/4:
df['file'] = df['file'].str.replace('/2035', '/2021')
df['file'] = df['file'].str.replace('/2122', '/2022')
df['file'] = df['file'].str.replace('/2131', '/2021')
df['file'] = df['file'].str.replace('/2121', '/2021')
405/5: df = df.dropna()
405/6: df = df.sort_values('file')
405/7: df.iloc[-1].file
405/8: df['lot'] = df['file'].str.split('/', expand=True)[0]
405/9: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
405/10: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
405/11: df.file.str.find("HIGHBLUE_20210924_103721")
405/12: df[df.file.str.find("HIGHBLUE_20210924_103721")]
405/13: df.file.str.find("HIGHBLUE_20210924_103721")
405/14: df.file.str.contains("HIGHBLUE_20210924_103721")
405/15: df[df.file.str.contains("HIGHBLUE_20210924_103721")]
405/16: df.drop(index=302124)
405/17: df = df.drop(index=302124)
405/18: df.file.str.contains("HIGHBLUE_20210924_103721")
405/19: df.file.str.contains("HIGHBLUE_20210924_103721").sum()
405/20: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
405/21: df.file.str.contains("HIGHBLUE").sum()
405/22: df.file.str.contains("HIGHBLUE")
405/23: df.drop(df[df.file.str.contains("HIGHBLUE")])
405/24: df.file.str.contains("HIGHBLUE")
405/25: df[df.drop(df[df.file.str.contains("HIGHBLUE")])]
405/26: df[df.file.str.contains("HIGHBLUE")]
405/27: df.drop(index=301043)
405/28: df = df.drop(index=301043)
405/29: df = df.drop(index=300511)
405/30: df.file.str.contains("HIGHBLUE")
405/31: df.file.str.contains("HIGHBLUE").sum()
405/32: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
405/33: df
405/34: df = df.sort_values('file')
405/35:
df['lot'] = df['file'].str.split('/', expand=True)[0]
df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
405/36:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res['-'.join(s)] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
405/37: dict_res
405/38:
dict_res = {}
sub_div = 1
for s, grp in df.groupby([df['sess']]):
    print(s)
    nb_day = (grp['date'].max() - grp['date'].min().floor('D')).days + 1    
    tab_res = np.ones((nb_day, 24*sub_div)) * -1
    tab_file = np.zeros((nb_day, 24*sub_div), dtype=bool)
    for idx, row in grp.iterrows():
        tab_res[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] += row['nb_positif']
        tab_file[(row.date - grp['date'].min().floor('D')).days, row.date.hour*sub_div + row.date.minute//(60//sub_div) ] = True
    dict_res[s] = [tab_res, tab_file, grp['date'].min(),  grp['date'].max()]
406/1: np
406/2:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
406/3: import matplotlib.pyplot as plt
406/4: ls DCLDE/
406/5: cd ..
406/6: ls
406/7: %run compute_quantile.py DCLDE
406/8: ls
406/9: %run ../compute_quantile.py result_both/DCLDE
406/10: ls
406/11: cd result_both/
406/12: ls
406/13: %run ../compute_quantile.py result_both/DCLDE
406/14: cd result_both/
406/15: %run compute_quantile.py DCLDE
406/16: import sys
406/17: %run compute_quantile.py DCLDE
406/18: ls
406/19: ls DCLDE/
406/20: ls DCLDE/HAT_A_04/
406/21: ls DCLDE/HAT_A_04/140519/*
406/22: %run compute_quantile.py DCLDE
406/23: ls
406/24: df = pd.read_pickle("results_all_DCLDE.pkl")
406/25: df
406/26: df.sort_values("file")
406/27: df = df.sort_values('file')
406/28: df
406/29: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
406/30: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
406/31: df
406/32: df[df.nb_positif==0].sum()
406/33: (df.nb_positif==0).sum()
406/34: len(df)
406/35: df.sort_values("nb_positif").
406/36: df.sort_values("nb_positif")
406/37: tmp = df.sort_values("nb_positif")[-5:]
406/38: tm
406/39: tmp
406/40: tmp.to_csv("detec_DCLDE.csv")
406/41: tmp = df.sort_values("nb_positif")[-1005:-1000]
406/42: tmp
406/43: plt.plot(np.array(df.nb_positif));plt.show()
406/44: plt.hist(np.array(df.nb_positif));plt.show()
406/45: plt.hist(np.array(df.nb_positif));plt.yaxis('log');plt.show()
406/46: plt.figure(); plt.hist(np.array(df.nb_positif));plt.set_yscale('log');plt.show()
406/47: plt.figure(); plt.hist(np.array(df.nb_positif));plt.yscale('log');plt.show()
406/48: plt.figure(); plt.hist(np.array(df.nb_positif));plt.yscale('log');plt.show()
406/49: plt.figure(); plt.hist(np.array(df.nb_positif), bins=101);plt.yscale('log');plt.show()
406/50: len(df)
406/51: tmp = df.sort_values("nb_positif")[-505:-500]
406/52: tmp.to_csv("detec_DCLDE_500.csv")
406/53: df['sess']
406/54: df['sess']
406/55: df
406/56: df['sess' == 'HAT']
406/57: df['sess']
406/58: df['sess']
406/59: df
406/60: df.iloc[500]
406/61: df['sess'=='HAT_A']
406/62: df[df['sess']=='HAT_A']
406/63: tmp = df[df['sess']=='HAT_A']
406/64: tmp
406/65: tmp = tmp.sort_values("nb_positif")
406/66: tmp
406/67: tmp
406/68: plt.plot(np.array(tmp.nb_positif));plt.show()
406/69: tmp = df[df['sess']=='HAT_A']
406/70: tmp
406/71: plt.plot(np.array(tmp.nb_positif));plt.show()
406/72: plt.plot(np.hist(tmp.nb_positif), bins=101);plt.show()
406/73: plt.hist(np.arry(tmp.nb_positif), bins=101);plt.show()
406/74: plt.hist(np.array(tmp.nb_positif), bins=101);plt.show()
406/75: plt.hist(np.array(tmp.nb_positif), bins=101);plt.yaxis('log');plt.show()
406/76: plt.hist(np.array(tmp.nb_positif), bins=101);plt.yscale('log');plt.show()
406/77: plt.hist(np.array(tmp.nb_positif), bins=101);plt.yscale('log');plt.show()
406/78: plt.hist(np.array(tmp.nb_positif), bins=101);plt.yscale('log');plt.show()
406/79: ls
406/80: tmp
406/81: tmp
406/82: tmp
406/83: tmp
406/84: plt.hist(np.array(df.nb_positif), bins=101);plt.yscale('log');plt.show()
406/85: tmp
406/86: tmp
406/87: plt.plot(np.array(tmp));plt.show()
406/88: plt.plot(np.array(tmp.nb_prediction));plt.show()
406/89: plt.plot(np.array(tmp.nb_prediction));plt.show()
406/90: tmp.prediction
406/91: tmp.nb_positif
406/92: plt.plot(np.array(tmp.nb_positif));plt.show()
406/93: plt.plot(np.array(tmp.nb_positif)>10);plt.show()
406/94: plt.plot(np.array(tmp.nb_positif)>100);plt.show()
406/95: plt.plot(np.array(tmp.nb_positif)>1000);plt.show()
406/96: plt.plot(np.array(tmp.nb_positif)>500);plt.show()
406/97: plt.plot(np.array(tmp.nb_positif)>200);plt.show()
406/98: tmp
405/39:
for key in dict_res.keys():
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24*sub_div, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.tight_layout();plt.savefig("%s.pdf"%key)
405/40:
for key in dict_res.keys():
    print(key)
    tab_res, tab_file, datemin,  datemax = dict_res[key]
    plt.close()
    plt.figure(figsize=(16,9))
    plt.imshow((tab_res).T,  aspect='auto', interpolation=None);plt.colorbar();plt.xlabel('Day'); plt.ylabel('Hour');plt.title(key);plt.yticks(np.arange(0, 24*sub_div, sub_div), np.arange(24)); plt.xticks(np.arange((datemax-datemin).days+1),pd.date_range(datemin, datemax).floor('D'), rotation=90);plt.tight_layout();plt.savefig("%s.pdf"%key)
408/1: %run ../compute_quantile.py hydro_blanc
407/1: plt.plot(np.array(tmp.nb_positif)>200);plt.show()
407/2: import matplotlib.pyplot as plt
407/3: import matplotlib.pyplot as plt
408/2: %run ../compute_quantile.py hydro_bleu
408/3: %run ../compute_quantile.py hydro_rouge
408/4: pd
408/5: df = pd.read_pickle("results_all_blanc.pkl")
408/6: ls
408/7: df = pd.read_pickle("results_all_hydro_blanc.pkl")
408/8: df
408/9: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
408/10: df
408/11: df.sort_values("nb_positif")
408/12: df = df.dropna()
408/13: df
408/14: df.sort_values("nb_positif")
408/15: tmp = df.sort_values("nb_positif").iloc[-100:-1:10]
408/16: tmp.to_csv("detec_blanc.csv")
408/17: df.to_pickle("results_all_hydro_blanc.pkl")
408/18: df = pd.read_pickle("results_all_hydro_bleu.pkl")
408/19: df = df.dropna()
408/20: tmp = df.sort_values("nb_positif").iloc[-100:-1:10]
408/21: tmp
408/22: tmp.to_csv("detec_bleu.csv")
408/23: tmp
408/24: df = pd.read_pickle("results_all_hydro_rouge.pkl")
408/25: df
408/26: df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
408/27: df
408/28: df.to_pickle("results_all_hydro_rouge.pkl")
408/29: tmp = df.sort_values("nb_positif").iloc[-100:-1:10]
408/30: tmp.to_csv("detec_rouge.csv")
408/31: tmp
408/32: df
409/1: import calplot
409/2: import numpy as np; np.random.seed(sum(map(ord, 'calplot')))
409/3: import pandas as pd
409/4:
all_days = pd.date_range('1/1/2019', periods=730, freq='D')
days = np.random.choice(all_days, 500)
events = pd.Series(np.random.randn(len(days)), index=days)
calplot.calplot(events)
409/5: plt.show()
409/6: import matplotlib.pyplot as plt
409/7: plt.show()
409/8:  pd.Series(np.random.randn(len(days)), index=days)
409/9: tmp =  pd.Series(np.random.randn(len(days)), index=days)
409/10: tmp
409/11: tmp == NaN
409/12: tmp.isna()
409/13: (tmp.isna()).sum()
409/14: plt.plot(tmp);plt.show()
409/15: plt.plot(np.array(tmp));plt.show()
409/16: df = pd.read_pickle("results_all_CARIMAM.pkl")
410/1:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import calplot
df = pd.read_pickle("results_all_CARIMAM.pkl")
df['file'] = df['file'].str.replace('/2035', '/2021')
df['file'] = df['file'].str.replace('/2122', '/2022')
df['file'] = df['file'].str.replace('/2131', '/2021')
df['file'] = df['file'].str.replace('/2121', '/2021')

df = df.dropna()
df = df.sort_values('file')
410/2: df['file'] = df['file'].str.replace('/2001', '/2021')
410/3:
df = df.dropna()
df = df.sort_values('file')
410/4:
df['lot'] = df['file'].str.split('/', expand=True)[0]
df['sess'] = df['file'].str.split('/', expand=True)[1].str.split('_[0-9]', expand=True)[0]
df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
410/5: df('date']
410/6: df['date']
410/7: df
410/8: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
410/9: df = df.drop(index=302124)
410/10: df = df.drop(index=300511)
410/11: df = df.drop(index=301043)
410/12: df['date'] = pd.to_datetime(df['file'].str.split('/', expand=True)[2].str.split('UTC_V', expand=True)[0], format="%Y%m%d_%H%M%S")
410/13: df['date']
410/14: dict_res = {}
410/15:
for s, grp in df.groupby([df['sess']]):
    break
410/16: grp
410/17: grp['date'].floor('D')
410/18: grp['date'].dt.floor('D')
410/19:
for day in grp.groupby(grp['date'].dt.floor('D')):
    break
410/20: day
410/21: grp['date'].dt.floor('D')
410/22: day
410/23: grp.groupby(grp['date'].dt.floor('D'))
410/24:
for s, grp in df.groupby([df['sess']]):
    detect = list()
    for day in grp.groupby(grp['date'].dt.floor('D')):
        detect.append([day['date'].dt.floor('D'), day['nb_positif'].sum()])
410/25: day
410/26: day['date']
410/27:
for s, grp in df.groupby([df['sess']]):
    detect = list()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect.append([day, elems['nb_positif'].sum()])
410/28: detect
410/29:
for s, grp in df.groupby([df['sess']]):
    detect = list()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect.append([day, elems['nb_positif'].sum()])
    ser = pd.Series(detect)
410/30: ser
410/31:
for s, grp in df.groupby([df['sess']]):
    detect = list()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect.append([day, elems['nb_positif'].sum()])
    ser = pd.Series(detect, index=detect[0])
410/32: detect
410/33: detect
410/34: elems
410/35: detect
410/36: pd.DataFrame(detect, columns=['date', 'nb_positif'])
410/37: detect = pd.DataFrame(detect, columns=['date', 'nb_positif'])
410/38: ser = pd.Series(detect['nb_positif'], index=detect['date'])
410/39: ser
410/40: detect['nb_positif']
410/41: pd.DataFrame(detect, columns=['date', 'nb_positif'])
410/42: ser = pd.Series(detect, index=detect['date'])
410/43: ser = pd.Series(detect['nb_positif'], index=detect['date'])
410/44: ser
410/45: ser[:,0]
410/46: ser[0]
410/47: ser
410/48: ser = pd.Series(detect)
410/49:
for s, grp in df.groupby([df['sess']]):
    detect = dict()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect{day} = elems['nb_positif'].sum()
    ser = pd.Series(detect)
410/50:
for s, grp in df.groupby([df['sess']]):
    detect = dict()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect[day] = elems['nb_positif'].sum()
    ser = pd.Series(detect)
410/51: ser
410/52: calplot.calplot(ser);plt.show()
410/53:
for s, grp in df.groupby([df['sess']]):
    print(s)
    detect = dict()
    for day, elems in grp.groupby(grp['date'].dt.floor('D')):
        detect[day] = elems['nb_positif'].sum()
    ser = pd.Series(detect)
    calplot.calplot(ser)
    plt.savefig("%s.pdf"%s)
411/1: %history -f -g NEW_CAL8.txt
411/2: %history -f NEW_CAL8.txt
411/3: pwd
411/4: %history -g -f NEW_CAL8.txt
   1: %history -g -f NEW_CAL9.txt