From 2f2136f4a027f806df40683cf4822f99dde9089a Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:04:59 +0000 Subject: [PATCH 01/11] Add json configuration files in installation of python module --- setup.py | 2 + .../.ipynb_checkpoints/__init__-checkpoint.py | 0 topcoffea/analysis/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 148 bytes topcoffea/analysis/btagMCeff/__init__.py | 0 .../analysis}/btagMCeff/btagMCeff.py | 0 .../analysis}/btagMCeff/run.py | 0 .../.ipynb_checkpoints/run-checkpoint.py | 101 +++++++++++++++ .../work_queue_run-checkpoint.py | 117 ++++++++++++++++++ .../analysis}/topEFT/Genrun.py | 0 .../analysis}/topEFT/Gentopeft.py | 0 topcoffea/analysis/topEFT/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 155 bytes .../topEFT/__pycache__/topeft.cpython-38.pyc | Bin 0 -> 18648 bytes .../__pycache__/topeftenv.cpython-38.pyc | Bin 0 -> 7267 bytes .../analysis}/topEFT/convert3lEFT.py | 0 .../analysis}/topEFT/drawSliders.py | 0 .../analysis}/topEFT/genPlot.py | 0 .../analysis}/topEFT/make_jsons.py | 0 .../analysis}/topEFT/plot.py | 0 .../analysis}/topEFT/run.py | 0 .../analysis}/topEFT/topeft.py | 0 .../analysis}/topEFT/topeftenv.py | 0 .../analysis}/topEFT/work_queue_run.py | 0 topcoffea/analysis/ttTest/__init__.py | 0 .../analysis}/ttTest/plot.py | 0 {analysis => topcoffea/analysis}/ttTest/tt.py | 0 27 files changed, 220 insertions(+) create mode 100644 topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py create mode 100644 topcoffea/analysis/__init__.py create mode 100644 topcoffea/analysis/__pycache__/__init__.cpython-38.pyc create mode 100644 topcoffea/analysis/btagMCeff/__init__.py rename {analysis => topcoffea/analysis}/btagMCeff/btagMCeff.py (100%) rename {analysis => topcoffea/analysis}/btagMCeff/run.py (100%) create mode 100755 topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py create mode 100644 topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py rename {analysis => topcoffea/analysis}/topEFT/Genrun.py (100%) rename {analysis => topcoffea/analysis}/topEFT/Gentopeft.py (100%) create mode 100644 topcoffea/analysis/topEFT/__init__.py create mode 100644 topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc create mode 100644 topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc create mode 100644 topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc rename {analysis => topcoffea/analysis}/topEFT/convert3lEFT.py (100%) rename {analysis => topcoffea/analysis}/topEFT/drawSliders.py (100%) rename {analysis => topcoffea/analysis}/topEFT/genPlot.py (100%) rename {analysis => topcoffea/analysis}/topEFT/make_jsons.py (100%) rename {analysis => topcoffea/analysis}/topEFT/plot.py (100%) rename {analysis => topcoffea/analysis}/topEFT/run.py (100%) rename {analysis => topcoffea/analysis}/topEFT/topeft.py (100%) rename {analysis => topcoffea/analysis}/topEFT/topeftenv.py (100%) rename {analysis => topcoffea/analysis}/topEFT/work_queue_run.py (100%) create mode 100644 topcoffea/analysis/ttTest/__init__.py rename {analysis => topcoffea/analysis}/ttTest/plot.py (100%) rename {analysis => topcoffea/analysis}/ttTest/tt.py (100%) diff --git a/setup.py b/setup.py index d987cfe98..da094cd84 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,8 @@ "data/btagSF/UL/*.pkl.gz", "data/btagSF/UL/*.csv", "data/btagSF/*.csv", + "json/*.json", + "json/signal_samples/*.json" ], } ) diff --git a/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py b/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/__init__.py b/topcoffea/analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41ab022a210617d05b2dec0b3c93f578319a79f5 GIT binary patch literal 148 zcmWIL<>g`k0?EGz6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Heenx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*F?KR!M)FS8^*Uaz3?7Kcr4eoARhsvXG4 I&p^xo02gp0X#fBK literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/btagMCeff/__init__.py b/topcoffea/analysis/btagMCeff/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/analysis/btagMCeff/btagMCeff.py b/topcoffea/analysis/btagMCeff/btagMCeff.py similarity index 100% rename from analysis/btagMCeff/btagMCeff.py rename to topcoffea/analysis/btagMCeff/btagMCeff.py diff --git a/analysis/btagMCeff/run.py b/topcoffea/analysis/btagMCeff/run.py similarity index 100% rename from analysis/btagMCeff/run.py rename to topcoffea/analysis/btagMCeff/run.py diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py new file mode 100755 index 000000000..41ea9282d --- /dev/null +++ b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +import lz4.frame as lz4f +import pickle +import json +import time +import cloudpickle +import gzip +import os +from optparse import OptionParser + +import uproot +import numpy as np +from coffea import hist, processor +from coffea.util import load, save +from coffea.nanoevents import NanoAODSchema + +import topeft +from topcoffea.modules import samples +from topcoffea.modules import fileReader + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser(description='You can customize your run') + parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') + parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') + parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') + parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk') + parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') + parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') + parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') + parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') + parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients') + + args = parser.parse_args() + cfgfile = args.cfgfile + dotest = args.test + nworkers = int(args.nworkers) + chunksize = int(args.chunksize) + nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks + outname = args.outname + outpath = args.outpath + treename = args.treename + do_errors = args.do_errors + + if dotest: + nchunks = 2 + chunksize = 10000 + nworkers = 1 + print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) + + ### Load samples + if cfgfile != '': + samplesdict = samples.main() + elif os.path.isfile('.samples.coffea'): + print('Using samples form .samples.coffea') + samplesdict = load('.samples.coffea') + else: + print('Execute as [path]/run.py [path]/samples.cfg') + exit() + + flist = {}; xsec = {}; sow = {}; isData = {} + for k in samplesdict.keys(): + samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) + flist[k] = samplesdict[k]['files'] + xsec[k] = samplesdict[k]['xsec'] + sow[k] = samplesdict[k]['nSumOfWeights'] + isData[k]= samplesdict[k]['isData'] + + # Check that all datasets have the same list of WCs + for i,k in enumerate(samplesdict.keys()): + if i == 0: + wc_lst = samplesdict[k]['WCnames'] + if wc_lst != samplesdict[k]['WCnames']: + raise Exception("Not all of the datasets have the same list of WCs.") + + processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + + # Run the processor and get the output + tstart = time.time() + output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.futures_executor, executor_args={"schema": NanoAODSchema,'workers': nworkers, 'pre_workers': 1}, chunksize=chunksize, maxchunks=nchunks) + dt = time.time() - tstart + + nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) + print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) + + # This is taken from the DM photon analysis... + # Pickle is not very fast or memory efficient, will be replaced by something better soon + # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: + if not outpath.endswith('/'): outpath += '/' + if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) + print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) + with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: + cloudpickle.dump(output, fout) + print('Done!') + + + + diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py new file mode 100644 index 000000000..05ed75686 --- /dev/null +++ b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +import lz4.frame as lz4f +import pickle +import json +import time +import cloudpickle +import gzip +import os +from optparse import OptionParser + +import uproot +import numpy as np +from coffea import hist, processor +from coffea.util import load, save +from coffea.nanoevents import NanoAODSchema + +import topeft +from topcoffea.modules import samples +from topcoffea.modules import fileReader +import topeftenv + +import argparse +parser = argparse.ArgumentParser(description='You can customize your run') +parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') +parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') +parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') +parser.add_argument('--chunksize','-s' , default=100000 , help = 'Number of events per chunk') +parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') +parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') +parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') +parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') +parser.add_argument('--do-errors' , action='store_true', help = 'Save the w**2 coefficients') + +args = parser.parse_args() +cfgfile = args.cfgfile +dotest = args.test +nworkers = int(args.nworkers) +chunksize = int(args.chunksize) +nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks +outname = args.outname +outpath = args.outpath +treename = args.treename +do_errors = args.do_errors + +if dotest: + nchunks = 2 + chunksize = 10000 + nworkers = 1 + print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) + +### Load samples +if cfgfile != '': + samplesdict = samples.main() +elif os.path.isfile('.samples.coffea'): + print('Using samples form .samples.coffea') + samplesdict = load('.samples.coffea') +else: + print('Execute as [path]/run.py [path]/samples.cfg') + exit() + +flist = {}; xsec = {}; sow = {}; isData = {} +for k in samplesdict.keys(): + samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) + flist[k] = samplesdict[k]['files'] + xsec[k] = samplesdict[k]['xsec'] + sow[k] = samplesdict[k]['nSumOfWeights'] + isData[k]= samplesdict[k]['isData'] + +# Check that all datasets have the same list of WCs +for i,k in enumerate(samplesdict.keys()): + if i == 0: + wc_lst = samplesdict[k]['WCnames'] + if wc_lst != samplesdict[k]['WCnames']: + raise Exception("Not all of the datasets have the same list of WCs.") + +processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + +executor_args = {#'flatten': True, #used for all executors + 'compression': 0, #used for all executors + 'cores': 2, + 'disk': 5000, #MB + 'memory': 10000, #MB + 'resource-monitor': True, + 'debug-log': 'debug.log', + 'transactions-log': 'tr.log', + 'stats-log': 'stats.log', + 'verbose': False, + 'port': [9123,9130], + 'environment-file': topeftenv.get_environment(), + 'master-name': '{}-workqueue-coffea'.format(os.environ['USER']), + 'print-stdout': True, + 'skipbadfiles': False, + 'schema': NanoAODSchema, + 'extra-input-files': ["topeft.py"] +} + +# Run the processor and get the output +tstart = time.time() +output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks) +#output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks, extra-input-files=["topeft.py"]) +dt = time.time() - tstart + +nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) +nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) +print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) +print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) + +# This is taken from the DM photon analysis... +# Pickle is not very fast or memory efficient, will be replaced by something better soon +# with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: +if not outpath.endswith('/'): outpath += '/' +if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) +print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) +with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: + cloudpickle.dump(output, fout) +print('Done!') + diff --git a/analysis/topEFT/Genrun.py b/topcoffea/analysis/topEFT/Genrun.py similarity index 100% rename from analysis/topEFT/Genrun.py rename to topcoffea/analysis/topEFT/Genrun.py diff --git a/analysis/topEFT/Gentopeft.py b/topcoffea/analysis/topEFT/Gentopeft.py similarity index 100% rename from analysis/topEFT/Gentopeft.py rename to topcoffea/analysis/topEFT/Gentopeft.py diff --git a/topcoffea/analysis/topEFT/__init__.py b/topcoffea/analysis/topEFT/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca8cca7a5e9d14632ec6078762204275a04cc61c GIT binary patch literal 155 zcmWIL<>g`kf|S1p6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6enx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*FCB;@KAq8}fhnU`4-AFo$Xd5gm)H$SB` NC)EyQ@@F7s002(`CL90& literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02a68d8510b6ab5cff39978cc1d501d3577f6c3e GIT binary patch literal 18648 zcmb_kd3YPwamV5&2!a$viMmA{mPJ|CLCLplOX8u!Iw+Br?6qVF#9M*@u>iEYAVvDE zd{A~S(@|n4X%aUPlemr3#!1_xN9?9a+NNplqq#O|d?ab3HvRg2ZNGkL|7m|Sy8r=7 zw&lp&Z)a!b&6_uG-ZgLWmUlNb#eDd?`0rml_5Szyd_O0m_AdltCvNk<0fbK|d7rMR ze#ONDYJhn`HOSA98scYI4f8XiM)}#G#OeW|+ClUc?x zvHWs>NoyFB-Axpv$jSwxBI3Pb*;8a zTZ`v4LK*ONwp>KreTSTIHd81XT4%rsi|kldXig+s7)#}|!U-0~G^5k+1V^%_uFnAWK{@xvu6TP*aa3=SSN;fJjk_DS`6EzLk-nr)`T-Toe<5H6 zg(Ce{=t7u!$_!x<5g;3=?3vJ15S*xp&XH(<9V8K&3C#ppqOndQEaD>0+QOnqGp zTSeDZdwiSN{w8UC(>tFB$mxWjA`5w)YILEQTEycwFk8myS~PZ8EA z1AuYi!(4dVAvHcw)(;OK*Rq)r%XGqKN-yO#rBs9-`g2we07IH8`h zu1`22L66CnLuGwPGdhaH9om>yAZ5MoGm;)j6$)D338c$dcFeL4hLO>nm^oounx3+< z>B)ryRlMWwLRvGdRJLGEIN@}$%*wjmCoN`r$}}eztkshboY3-ly6v?c>-T8KIy)}5 zk$GQX^9opa%*lECHLF7}=dElhpG{}2iH?gDOgQ1f5Y{1cvXvfl1Spu)e_MUwT_o~fBoZFaYo(F}D@(}-2JQPF0C~J>r4!V)AaTh7 zEhVyrOb4y29j>HTk&BDTgrHt0rk9I_dNC)MtCK*DJLX7)v@M#Inw(!>)(P;|UR;4f zGPv;<{XzxV>4fz9EN;1+#R=u=)7%o>H2;V-*|c{pRoJUW_E%WGJLee-QG&6Qz$yZ3 z2&^MOPBk_V*hpX#fz1TA5+HvX+X&o1;6?&B0XU7^wr&ezMUN)kerq~S$?Q)vQ~OJ)mME14w!ndBj#5>~>&pt2g*{@1wFe~-1rl*uKR z#5F6PU6!8G=PF}M7Pm$(CIm9iIKTDB*ZX6qnIfB%Mi9&WDk zGW1QUI4+Kn1H*5LBp2g+PU?R2Wo5L?{TQ093>?VLK|L zJ-(Am>;~B&{i9(+NdE;tLOn$!AniXbP=3IU0S4_xz>pmW4BJhB5j$Z;Wjxn_JBGUv zcYG=+bz;BaQ#-vCfSPc6_&xuCfVZR@NN^^2`Sjw#GYyhXB&pYZo(>bc~y?YY$IlnJqN%75N({6?%S_(g}k zOh(QJ%rjz@Oyo8!@&P8=0LAJZz8cXbR@}GY z-hun}sVKCdMd4ZWp{keB@2g%;u$L2%rg|*oc3jE6TnS%xz(aS)mE04qMY*da($L#o z&H~}~_SG^d){pwn2aScI(_T{uiVgM}$lNK{j0Wws*5(U4p=F7|<12mUU( z3fQ$(Y<(5$gm**cQ&f_@0o>idnjw1~p569Ft4CfZH^_}s0sHz2E>?-_ri1pTD&5BP zW~~5qVmb=7q(IvKJ{9^lb!0wSfxqj5WCwHV;_g?Oa;1jpqm%C2fun2uww2b=f zKG{ch*?U;mZs@vE+~n!nEBEG(Lf78hKvma1)&)Ls=jhrm_p`3MWRL8XecW;f7~jMA zLAh7%llvJz#Q0r|ANJbh0JVvSA0!-g?DyzHHED;vv?K66)i%M9+Kg&V)RCGNJPIkQ zM}i@>G1Z5t0d8@(od;f5m)qWhudB_KyZd$Zx&7nV>*{mmj$4D`X4x-qmq&&J3-J<6 zp}BcI{M<|P68G_5=IoREs+@hyK^s>&eazV-_f$D(dFVmgRXO{abCPq|E8HJAL+jZ$KV% zh<$=0~tJSp!%|GOXULzX2NQXeA=iE{hcJ;3jA`_WX) zPF}e*&-;z`sW*^=R)$79RWldD|8wSyp_+NrO|6|r)9m@7x=}8q;88BTQC{&pxJa2Z z#^Fir4ak${huP1X)T(~gm@_PgtDIrd>6T_^WJadp`4O4PU1w+INx+=EA8=HT0OsK_ zvN*wz#xq$?lslf&z|-#dM~moJF735Q;SJ<)TcpT7N3?d?Ilfdg$3tdLd!6PMDd80+ z=5TRKCcv|}!Q9i}Vm21HdR)_!t)1^JPqKF2E`xWsEY*##G3K4FYx!~1)Zo5Nkova4 zeVd?59fL28+_9=R&mf1%z9&e&H`(_DUFtY;n7QLj1>ygRH&%w%29s~B952@cZ?G)4 z_2e5Y%gtrWGS5$C_~D)z#lr^(M?WC$en707qXAyy&cgrqvea4h^s{y2>bB7!YQS+R zkBqbK9E-8@Aabd%nhO!To6CMkS}bLu9V^yYXhqi=;_jD3SFCxsYR$u#@gANtzt&sx zaMhYes@7aC*Q_;DT#G3=RUO~wnDdByq{?}eISR!gx24okw@bV9E>n<-YAMk_To^i)$$bK3# z4(^&r@^>*qJzW=3e+Yg%0d5?z9`79N56cgOcT#?s_f5BpJNTWz1$hBKZ~7j^?oRA@ z)IUt`NoM4XwOh{Q?wL}=oukD0NDXHvb3V$x`3R!%k1_U9V9&^p%8$`5Rz5=zGvP;T zZ2Ud>kvXX!sY(5P*mhX{e%+k*apwJA-JJF;M)SvcP9sRO*vENJBS_Sy6K_H$FLME&VJk`YiVag4Ft-<-S1BrD!GlthbVV2KZ;#OVmgH^ta9HdE?K1 z>-?@a{+zu*yT)h4j!VYTU$8g8N2zaAq)(=Mf+=6pZ#@l0B^th?QdA=oVb9~A`&}!{5<(}`3<}o1YGR9 zFf0y|R^tizwL(mWWO%7heyz!O=;9+pStpgcx= z@UZ)Zw!`ucd8gdz@p?SofV^8C_juR=v)mauDW%859+-I(@&RdkJnWsBr^|vYdc3_J zFDXySl*jA$cwdlTlwb0AM?BtN%FoNc@_0u*-k-`plYj2<20Y%U~D)>;<&g++$+>f$o`J}HtKdv_Q_tXy??#a^*8a3%RQIy?!h~OeUST`8o7JrU6=4w zd7J!OS4P|?PKcA@{!GyRE|-Pe(bjE=#x^De_xY+zZ|a8-xW!5N~FY4CU}$2 z)0dRzy^#NQTlphqOr=v(Yjg%#<-(aj6JQLVb=2?sp zuMBK8^Trq>-Z*|=sF}gQIkN-5lG{I#5pg?y6?DcmS_QZ20{rg|IlAQIPJ- z)lYhS=YuCd zWdBS$7fLferXD}xw_mhhlHZZvZTHFVwYyxyepx)i>B8=o|G?NW`xW^!`{(j!Vrn!D z+rErBfT*)<=}V}|>B>B7|DteAzAV4Xa=(!;BYI!)D8=Z%EGz^6JK!%jTA07w;|Jx- z_AfCXFE`d#>Lh=Tc_fax_?J{eqU>MDmotidC4;xJvYVj&Px9Njd*wep;j{l){zCpU zB*_oI!Y@R-Btb)*!>=zFy*X;R@euq0DW1Xp1)iaqYq{}W`K>DVU*(IUh&nAd{tFg9 zN_8-wgVo<-t3Q8!+3DvBt(XlD%9rHJ@G!K&vR}jVMfUu^$bZGbmzU)$vRnRI{t6y? zY`OvK2l7GlX=q+ykax%nta(luD~t}3y=)1mU!{b%R7-e??142eicv1nZ>{EfQRGZKSRinj%mJZ`qXn8lZbQp;%*7782DSQ=eYqE~Bo+TZ|(JR*R6zg~lI?Al$ zOI17Gd&N55!#Z|D#~5@hGoJJO+F>|XtmC~Ek08g#aDM>zIoxCNr#LS5zC%t#q^y*w z;h+qiA>MT+YaYU38z(rSr3@!{#?;bIQ=$GyS=Q`99yKLgLIUQ zlJ^vg;^P0|Z;;MBr(mlS9o4jw$m-_BW@O8J;QjRX?46Wf7|?*T(q?yFOBIZ@v*|y+ z)1cE>7XtulxMU>@Mcwh2EMrlHVWh^@D4j}65=J5!t>6TVks=PS#PX@hiDWvJ9>LLS z<2dqUUyh*gpZw3++zZzC?#%pR>OUn+MQ7*PA#L?j6hPxKGnIc4&He zBM!ak<&zz^cieYPc4LQopM<_HWbDd7KmOM4yodwPzCIL7mOBk8Gp!Zq(AvdD&=;GC z>DRU)3vNxr6TZ=OC!m1{=%9cB=;$ROJ{agUz&Y-LLmb@-r1E*A2rWj5z-a;o0h545 zpbUT$bEJ@O^$iV=+RkpavF+cqt7oMoCdvyQfAHRPU_qf>CRC+IqxjE9Kv zFo8!1Oc6Lo;86mP5qJxMw-R_8fwvQQ2Z6^4tO4kZUa>eI*W;6Vd_Iql>+vByKC{Os z^;SWn6Tx7~;%u85;YPW;q&6_pQyeciQMbXsjJX(b)o7vU9y>M8kYRLourp*VsNgtT z$%j&%20E9Qrbcv{2*t5HmqcfsCg~^KoagaSL;HJ=qMQgGTnOj^mmVM*LntJyiIUbC zbREM!VLy?JIvb29N#j!l*plUh(fLVZB?0oD@ooYg0Iw^~-n+awx9sJLxV$)@oxF1K zb!6g|i?4rOaf43$b;fGk!$rJ)#?FX)7WB%E;ZF71;-Q+}&n?LPo92MJCgFbjdYT@h zy7P$Qk-~j}rj$BO-0xpoQ*GDbv2x|&H;@TeF3v;mwZ*Xn;#hNvR&nDNXm>&;jz!}L zEY3MQ(dr@_q=ncC45rFXV83RmF?X@$#XBI*OsIY_sfNsKX`8^nE*4E~pHLgUwU?xf z6Qn$VqSQob_^6iOXBM|?OJcHD!}?fiaB#1h&^3`Q>-Y#qFIkk`qJ_KLJsR#$jXGf* z64z5_jJwGTg8&sX$V{~b(@M6#&sEKeqxqs~?agA=tXRT=36$4|g16putP57C(R8se zc1*L1Q~@oksezHArZ#E&3S$Seg|gWX#89cY2DjGmf;taDffyX^w+`bIDlqyjHJTN> z3Rd=D8FGxFG&PI>fC`zH6+K!yHF2;gv_sl>&q31(6w1028A+Kb%QDmk2=?(uQCPwd zIXF%9Ar+1-r%alLoQC0SK95MFps9^GA&=lea8Thdl$^*gzSgp~bvZ3V20q(LBTF)6 z7^w+UT{xUQqlrC6RvaBE_8|ew$F6)H$LJ~kFpU;!me5c+n-}Cwj+M~BP9%e-BD+G! zfW`Uf6IBTS}8m~1L^7YnA9waUD925_WbZJ-&v2jSIj#9jq>2}k{5 zJjY(!^Cr%)DdCjurum$QrlWz;ESV+n0HXk;@P%~0c( zi}V$Z?Rn!?q|@gMbi7+lRL*xNtzt26IuSPjLa#^(;Y6)s61}4|VeBH!J4tgQoyw<^ z<6Z8OW*QW38T$bobUe5nUo!B5;{>UvIH4@Qq%iS`A6)JPP~Fad#AI7s;q z5jaespFlf-qX16Si|b8C5pINWir9V~6M%(TK*^~w8UhIXl6qN<=;dSy(KjvTE)!i} zb6KGR54%u-XNzEghEK#AFCNfJkhew*O=FaJEJ+B@8QgZ%Y-$5~VacL%{A%2_-wmMQ z6&&Bkt^uO~BBmP6Ajn4tHr-%|7Ei~Y+oEzPIF7g?YwkwBBnh<%o0TGeeaIPjiFyzj zjCWCyUM$j60`(*>7lx9h z(2oaYt{P0NxN9NpLqJu`M@o(PZek6Cl;JB31d5oHsqUTxv!Xf{pCqDkR0^vEV_{9> zXQTII`x7p4xKc$_m#WBuF^prVX55Wd8TG7%aRO8;4muA-ilpCv1+YZd)+YdP5JeJtJsWu~QL&rDS zcJxcNrH+A_a#n=uR42j=5{XwcLBxrUr40TYPmR*Yo(J&KPz{b0%a+ZqR+n7iHeRMxbcS?4kF5{+N=eUM)&giTYXs_zKUxf=iSel{WuEB?h@zV1v@9 zv?|Lyx+T!1bSQB*4KNNI`IUvr!ay5zwBcHzEK*Ryf=z)Ar3F}vvd|y%g_R(1@B;*6 zRF*FeZXD??u!gim{p%IJ{H-n>SpjV=&>2+{{vhxUr6T~Izr#-@EklmVlPsZY8Ejpr ztaJ4!>;0{PR@h&?qNK6SzeU+XJlG%gZ&JFHF4o!x?kwN)M@>3v$u&foe&m6l+-nP- z>!A~{Qd(fGver*^BKs;38Ur2=HEbmxMB{8nD{Cem`J8P;zMV>s(&Hy;p|S(i&P(t` z@HI=hFfiM$gtde=27+ArYGo07imaJ)1y?Am(Q=6@zXe{QHo@ihp1>01S@$9?=r)#t z1+%51jbbQ|TGFq)%JrYImN_ZEiN)ML;lftfj9%jpqyM#_HDlx<@{2zRoYMj^UqYdq zz8j@9vY7f{Old=(YE{;Px55`B46P9Jfv()>Lz_T#kVZ(v{3JUUTV3T{4%@CwUgn!j zZ=BVAlj+BBQ*d|VHtFTF#J@M;FF|>j;^p3MFnKF9cic9X@CLxsnSmbnYf$BKU!vAX z9_nnMk8Ek0N+$7zZ&ELca-Q&bGI_e3%DXAX9u(_Dp&tLH1e-6~ix~$9e4oG%2=vy2 zJ|f!ymf$w&MJ5L2azDB&?qKvmpL=a!8L~mgNAJnxWMb21ci-5DIMm!cnee_a-sEyS z*EtRO$?cnl4f^dRaSvZ-0&FQM?m!x0%ln9!ZUl zj;9P^e2AzpwmW!(%AZBM-%VrhOMBN%WhD1yvvp6{f8O% zlkfRtf4sl)Z(nxp?q<&$KctF8uqu^O*q>6z!(P4!zj^;4=W^}pM0dD#x`R5bxdp(T e*Lf^sJi9Xi=2wg+%(_ZTq_xuEgHeq7xbpv{0XPK! literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92ffd4d8773d148881fbe15f61db0e2edb605323 GIT binary patch literal 7267 zcmbVRON<-Id9LaQ*=){(Lye@7G#a(2hnE|1^(J8Rf$Z-696g98Ks6c{<=;Dc`oa>+Fa5};23a?&}61e1sJ{ng|Q zXJk7_D0NkJbv^(3|L_0)nh$4ZZH8a-AOHH}f4acfztGF{&%n#uc*6fiBADQ9R!wtW znfosytK8s61QEs@$qtD$iAODz~e)XN~fEtUBXatyw&CLKFID z8fU_&&I_|z@Yra7CKPenBE9Rc_>M1cV3aB|P7DN&G zk~kq2k)IJK#VO=x#c8pG{G7Nb&WUC8Sbm3z^J3*W6D!{NUA?;ET@V*Gwa=N{_Z)9| zm+f+_gnz)wrI(NaTN_-t@|qjgBfsO7!<$!LYkQBqcGn%hb-XZioA^@J5>tIjGL1lX z+^A}J-N)hJ=k`3p-)+le-U?(H{L z-&%d6bj>NHogQ6NgHn6?rL`4I-1O3Js#&Gy);Fn>@Y`+5w`CASA|0mD@Z8d)gJFiM zG)%`W^+Qh{S=o4AtmvS})EY`XzlY{i-_p_Sp~jzktNI#6PZyi z*7q>V9_K8wW22R)D8(w|07WAWDKT0wmJiPb2A3jWqcinc|6Yr;u)aXgJ9WW|&x*qNYmR zd%@&6EOP%#*EfQWcfH;TE3M%1j@!K+1-*JeI&t0Yy6v6N52@<@gNKxPjR+I0_I8qD zP4>GrB`c(An%6L3IERGs6;0!dyvR>+o6Ga4sgp=1KPqSO+{F`KLlUtO7ks1%ZKTKA z9yDMNDbCA?nb3t1^M(;?Kh}OoXJoDoF8oe6jIbNX1cG-UjezpUzU!zJ?el9%)=%5) zYn3_#lObVxGBJg>-fyZCk_(txt*)#m7WP%^xzPrsw&8ZWUOO>*o6S&(DNUBqJULw( zk2~HwSr+U*yg_>@B4K=qYm{c1<7@i6px=C`fOs?@NykK3t8qqrv%KE+klKqT~D$E!>a} z<{x|sRgj~SsfKGz=G!Fkavm z)NL$xrZMfm(p_g+{$6p>}RCObik+?**MnEReNiyeLgmJ;c8TU4)qGV z2(MyY*t@iLYMoLj zdU~`ZinRAJidxUC35rH%MrUdDSn+sI9i-R$_Mvm# zw3W+l3yrEnUyKQhQ=6_n11ASLWMU<{bimS%}_E+34yaDkdh}TA=8m3k-)EQ zz>l?)jO;|x^UAqob~0#f!wol*+{8}9#KghWz3S}2>5;Lg174;9uTWB<_TkgrUeANW&V+%CJW)2P77bdH?i09u$Qhm8nGc8%8ywPBEXiwK|rLjc8hf_0F^3z#I`KJP8M|T zF9134Y=dRhEB9)zG3=PnOoE*)fX|i{(~;O~iQe?1K~8#)!Fgo}LA`tbEBDGg(d$o0 ztdSe_Q&vYx+;;u$V6jr^cOwZK@d{3fSV^Q`NGrT7Y0}E z2K}~hxFD3l+lZt1`&Il+2nD%7~zlZ7EAR zqG@CrWU7Qg%6&IskdJ1D{8JY5GwhGFE#reJ+<^1YR==3=-U}|ji7r#rz`jNF8lvgl z`HTij0YGDeOC$;KYi}%()mzFa!aFETddS~HmxM_&f|m-V%_K`;M_dMg=-znqHxxw# zl$Jg+A`{#YBBn42Q7bHPs2%%ms_&VD?>Ho|MFnwywyQ(fV$3(Q z^1UcC$_fNa2ZTjKzyv@9VR#P~`HT8|@+w_5{^yuOHbM$(Xc>bXkQ6iVs0k zT3Q~JFZkLEJ<$T78(C;#fgp4!(1t(fjpHuFy47|&>%zVBCm8Kt@vy;4t={%r#3SV9 zrw==ANcO+!?F{TjA0ZI=-~ZF{WE*qH4kcTZ9EDvOqV7+rK0-3M^icZ101ynd81V{i z01GMG@_n>TjAlDnmtE>hl1T%STU1i;a6p-X8eS)O>>-Y^)u^@5AHW5VQAg3{u$r3) zAH~T=Ln&#vqbp8krp5KgXfS5c4tL;WEx-f7LFJMFTcEuRK)Ile(=r9xj#b zG>k&TqoPKNp`8kN>Ez^UAi?VhY==&z?>rq7MZ8qTGmlrI?j}&Hn3z+#JXuA_#%d=a zo7yJ6I12h4TAGdU4fFl~S3$!~#!ok^?spJB7mjZs$&Qz=RC|y+5a<*wE*#f$3;eZ5 zJ~%0^L8{^*UIXdMod;BBK9L}}AfW9&;#N{Z#cdOVPT*HF+_EWSYCoie$Z1s_o%HH{ z&-*3LbiC0+XS#=eV`o|bJ+*R(Gnw{pmQ9ZJEaFdGcI=`}k{!n-FA4LGSPyY=TDWqp ze*;}hDeAEh6j48+t#YLnNGPsA6iwBm+%9Z^F0^*}_f&}WDceOLNP!}-o$cxf3CBS# zuDZ<89O{)fXVogpDtM9xcSyF8?-N z&gH44;6oM_M<>M0Xim(I77<3`J~p<96cDo`i;S84chzPdv*WUj>J;3g+&YPXxNr~< zqh@q!k0H!-&;xh3Q$3cjMhb}$Q0}oG{a^_H1Rqu$+R-_zKNs7ygU|Was_I!h?1=z( zxpkiIf8(4u8L4p);4ZgT#HpzOce!;z1-O3+bjd4papHXIqC%I_=#n@+g)aGL`AL9# zh%V{Kpf^iv59M{7sbUh+LuoKK!C!>7_%CnJXW|=2yy6ieM4Ukpp~VRi5)2}T!%hiN znX3U(D6gUM3tr8W;it@SOh0H8Sx54>ctYaz8R+N?+!9zSdA`Zz;IxyRFrvh0xPDtc zN3X$~Uq1}3r@A@oUfso|Fs_DPCB_J^wQJ5RTO^O69RCsuYKw&-R~iA`XlYY?547qa zPL|(BYe^eD=DpPR{toqrbnoP``+)Q^K)ykp-=yR%N`8})o0NQ+5?WRoonM}sRe9gr z^r=Bf6UkI8+zNxPq_9f*lu!_*=(D17Dj+=!|fnM{uZ@- zn-b;nRcQYVuSxL$0Po>$uO0<*MtjumYNp*;An z`e%S|q``hc1^YH}j;S#JDwp5KIHb=mXLF)yn)qS%08R zM1BVi$`^4dPfDS*rHmSRmlEZi7OAAp(I7G_=jr`1B}Dm>97Oh>x|LS?kkEaYU%wl4 z8-6n}@7@3H58shW9Y3YkW;<|2D8EA8=|+9(mgil1J*JDq+fr2_3Cb!Zv^DtyO4J?D zAt^lAePZ28MRkXEAClRmCv_~v8RDA8FbcK~@0YW3_GN3qDxx%NowQ#s=r;c!#u58% literal 0 HcmV?d00001 diff --git a/analysis/topEFT/convert3lEFT.py b/topcoffea/analysis/topEFT/convert3lEFT.py similarity index 100% rename from analysis/topEFT/convert3lEFT.py rename to topcoffea/analysis/topEFT/convert3lEFT.py diff --git a/analysis/topEFT/drawSliders.py b/topcoffea/analysis/topEFT/drawSliders.py similarity index 100% rename from analysis/topEFT/drawSliders.py rename to topcoffea/analysis/topEFT/drawSliders.py diff --git a/analysis/topEFT/genPlot.py b/topcoffea/analysis/topEFT/genPlot.py similarity index 100% rename from analysis/topEFT/genPlot.py rename to topcoffea/analysis/topEFT/genPlot.py diff --git a/analysis/topEFT/make_jsons.py b/topcoffea/analysis/topEFT/make_jsons.py similarity index 100% rename from analysis/topEFT/make_jsons.py rename to topcoffea/analysis/topEFT/make_jsons.py diff --git a/analysis/topEFT/plot.py b/topcoffea/analysis/topEFT/plot.py similarity index 100% rename from analysis/topEFT/plot.py rename to topcoffea/analysis/topEFT/plot.py diff --git a/analysis/topEFT/run.py b/topcoffea/analysis/topEFT/run.py similarity index 100% rename from analysis/topEFT/run.py rename to topcoffea/analysis/topEFT/run.py diff --git a/analysis/topEFT/topeft.py b/topcoffea/analysis/topEFT/topeft.py similarity index 100% rename from analysis/topEFT/topeft.py rename to topcoffea/analysis/topEFT/topeft.py diff --git a/analysis/topEFT/topeftenv.py b/topcoffea/analysis/topEFT/topeftenv.py similarity index 100% rename from analysis/topEFT/topeftenv.py rename to topcoffea/analysis/topEFT/topeftenv.py diff --git a/analysis/topEFT/work_queue_run.py b/topcoffea/analysis/topEFT/work_queue_run.py similarity index 100% rename from analysis/topEFT/work_queue_run.py rename to topcoffea/analysis/topEFT/work_queue_run.py diff --git a/topcoffea/analysis/ttTest/__init__.py b/topcoffea/analysis/ttTest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/analysis/ttTest/plot.py b/topcoffea/analysis/ttTest/plot.py similarity index 100% rename from analysis/ttTest/plot.py rename to topcoffea/analysis/ttTest/plot.py diff --git a/analysis/ttTest/tt.py b/topcoffea/analysis/ttTest/tt.py similarity index 100% rename from analysis/ttTest/tt.py rename to topcoffea/analysis/ttTest/tt.py From 36b02af50d1143f0d365be872b3adf9fd56facd1 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:08:09 +0000 Subject: [PATCH 02/11] Add UNL specific file provided by Xuan (works on coffea-casa) --- topcoffea/json/TTZToLLNuNu_M10.json | 32 +++++++++++++++++++++++++++++ topcoffea/json/__init__.py | 0 2 files changed, 32 insertions(+) create mode 100644 topcoffea/json/TTZToLLNuNu_M10.json create mode 100644 topcoffea/json/__init__.py diff --git a/topcoffea/json/TTZToLLNuNu_M10.json b/topcoffea/json/TTZToLLNuNu_M10.json new file mode 100644 index 000000000..1498dc810 --- /dev/null +++ b/topcoffea/json/TTZToLLNuNu_M10.json @@ -0,0 +1,32 @@ +{ + "xsec": 0.2529, + "year": "2018", + "treeName": "Events", + "histAxisName": "", + "options": "", + "WCnames": [], + "files": [ + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root" + ], + "nEvents": 14542666, + "nGenEvents": 4876491.715182641, + "nSumOfWeights": 19992000, + "isData": false +} \ No newline at end of file diff --git a/topcoffea/json/__init__.py b/topcoffea/json/__init__.py new file mode 100644 index 000000000..e69de29bb From 489a66210e9e9a7304ca70c5f0e22457fd95bce0 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:08:48 +0000 Subject: [PATCH 03/11] Add a coffea-casa notebook itself --- topeft-coffea-casa.ipynb | 368 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 topeft-coffea-casa.ipynb diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb new file mode 100644 index 000000000..3608c1de9 --- /dev/null +++ b/topeft-coffea-casa.ipynb @@ -0,0 +1,368 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "f467b886", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Obtaining file:///home/cms-jovyan/topcoffea\n", + "Installing collected packages: topcoffea\n", + " Attempting uninstall: topcoffea\n", + " Found existing installation: topcoffea 0.0.0\n", + " Uninstalling topcoffea-0.0.0:\n", + " Successfully uninstalled topcoffea-0.0.0\n", + " Running setup.py develop for topcoffea\n", + "Successfully installed topcoffea\n" + ] + } + ], + "source": [ + "#!pip install git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\n", + "! pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "781aeeb6", + "metadata": {}, + "outputs": [], + "source": [ + "import lz4.frame as lz4f\n", + "import pickle\n", + "import json\n", + "import time\n", + "import cloudpickle\n", + "import gzip\n", + "import os\n", + "from optparse import OptionParser\n", + "\n", + "import uproot\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.util import load, save\n", + "from coffea.nanoevents import NanoAODSchema\n", + "\n", + "from topcoffea.modules import samples\n", + "from topcoffea.modules import fileReader\n", + "\n", + "#FIXME: analysis is not installed anywhere (should be installed as well)\n", + "import topcoffea.analysis.topEFT.topeft\n", + "\n", + "import importlib.resources\n", + "\n", + "if hasattr(__builtins__,'__IPYTHON__'):\n", + " import sys\n", + " sys.argv = ['']\n", + " \n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\"\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0286ca63", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], + "source": [ + "import argparse\n", + "parser = argparse.ArgumentParser(description='You can customize your run')\n", + "parser.add_argument('jsonFiles' , nargs='?', help = 'Json file(s) containing files and metadata')\n", + "parser.add_argument('--prefix', '-r' , nargs='?', help = 'Prefix or redirector to look for the files')\n", + "parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks')\n", + "parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis')\n", + "#parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers')\n", + "parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk')\n", + "parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks')\n", + "parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms')\n", + "parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory')\n", + "parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files')\n", + "parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients')\n", + "\n", + "args = parser.parse_args()\n", + "\n", + "if args.jsonFiles is not None:\n", + " jsonFiles = args.jsonFiles\n", + " print('jsonFiles {}'.format(args.jsonFiles))\n", + "else:\n", + " with importlib.resources.path(\"topcoffea.json\", \"TTZToLLNuNu_M10.json\") as path:\n", + " jsonFiles = str(path)\n", + " print('jsonFile was selected for UNL {}'.format(jsonFiles))\n", + " \n", + "if args.prefix is not None:\n", + " prefix = args.prefix\n", + "else:\n", + " prefix = \"root://xcache//\"\n", + "\n", + "dotest = args.test\n", + "#nworkers = int(args.nworkers)\n", + "chunksize = int(args.chunksize)\n", + "nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks\n", + "outname = args.outname\n", + "outpath = args.outpath\n", + "pretend = args.pretend\n", + "treename = args.treename\n", + "do_errors = args.do_errors\n", + "\n", + "if dotest:\n", + " nchunks = 2\n", + " chunksize = 10000\n", + " nworkers = 1\n", + " print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "79e4d98b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], + "source": [ + "\n", + "samplesdict = {}\n", + "allInputFiles = []\n", + "\n", + "def LoadJsonToSampleName(jsonFile, prefix):\n", + " sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:]\n", + " if sampleName.endswith('.json'): sampleName = sampleName[:-5]\n", + " with open(jsonFile) as jf:\n", + " samplesdict[sampleName] = json.load(jf)\n", + " samplesdict[sampleName]['redirector'] = prefix\n", + "\n", + "if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',')\n", + "elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles]\n", + "\n", + "for jsonFile in jsonFiles:\n", + " if os.path.isdir(jsonFile):\n", + " if not jsonFile.endswith('/'): jsonFile+='/'\n", + " for f in os.path.listdir(jsonFile):\n", + " if f.endswith('.json'): allInputFiles.append(jsonFile+f)\n", + " else:\n", + " allInputFiles.append(jsonFile)\n", + "\n", + "print(allInputFiles)\n", + "# Read from cfg files\n", + "for f in allInputFiles:\n", + " if not os.path.isfile(f):\n", + " print('[WARNING] Input file \"%s% not found!'%f)\n", + " continue\n", + " # This input file is a json file, not a cfg\n", + " if f.endswith('.json'): \n", + " LoadJsonToSampleName(f, prefix)\n", + " # Open cfg files\n", + " else:\n", + " with open(f) as fin:\n", + " print(' >> Reading json from cfg file...')\n", + " lines = fin.readlines()\n", + " for l in lines:\n", + " if '#' in l: l=l[:l.find('#')]\n", + " l = l.replace(' ', '').replace('\\n', '')\n", + " if l == '': continue\n", + " if ',' in l:\n", + " l = l.split(',')\n", + " for nl in l:\n", + " if not os.path.isfile(l): prefix = nl\n", + " else: LoadJsonToSampleName(nl, prefix)\n", + " else:\n", + " if not os.path.isfile(l): prefix = l\n", + " else: LoadJsonToSampleName(l, prefix)\n", + "\n", + "flist = {};\n", + "for sname in samplesdict.keys():\n", + " redirector = samplesdict[sname]['redirector']\n", + " flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']]\n", + " samplesdict[sname]['year'] = int(samplesdict[sname]['year'])\n", + " samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec'])\n", + " samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents'])\n", + " samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents'])\n", + " samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights'])\n", + "\n", + " # Print file info\n", + " print('>> '+sname)\n", + " print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO'))\n", + " print(' - year : %i' %samplesdict[sname]['year'])\n", + " print(' - xsec : %f' %samplesdict[sname]['xsec'])\n", + " print(' - histAxisName : %s' %samplesdict[sname]['histAxisName'])\n", + " print(' - options : %s' %samplesdict[sname]['options'])\n", + " print(' - tree : %s' %samplesdict[sname]['treeName'])\n", + " print(' - nEvents : %i' %samplesdict[sname]['nEvents'])\n", + " print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents'])\n", + " print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights'])\n", + " print(' - Prefix : %s' %samplesdict[sname]['redirector'])\n", + " print(' - nFiles : %i' %len(samplesdict[sname]['files']))\n", + " for fname in samplesdict[sname]['files']: print(' %s'%fname)\n", + "\n", + "if pretend: \n", + " print('pretending...')\n", + " exit() \n", + "\n", + "# Check that all datasets have the same list of WCs\n", + "for i,k in enumerate(samplesdict.keys()):\n", + " if i == 0:\n", + " wc_lst = samplesdict[k]['WCnames']\n", + " if wc_lst != samplesdict[k]['WCnames']:\n", + " raise Exception(\"Not all of the datasets have the same list of WCs.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dd0c41e4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[# ] | 2% Completed | 0.1s4s\r" + ] + }, + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'topcoffea.analysis'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mtstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'topcoffea.analysis'" + ] + } + ], + "source": [ + "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", + "\n", + "client = Client(\"tls://localhost:8786\")\n", + "client.register_worker_plugin(dependency_installer)\n", + "\n", + "executor_args = {\n", + " 'schema': NanoAODSchema,\n", + " 'client': client\n", + "}\n", + "\n", + "# Run the processor and get the output \n", + "tstart = time.time()\n", + "output = processor.run_uproot_job(flist,\n", + " treename=treename,\n", + " processor_instance=processor_instance,\n", + " executor=processor.dask_executor,\n", + " executor_args=executor_args,\n", + " chunksize=chunksize,\n", + " maxchunks=nchunks\n", + " )\n", + "dt = time.time() - tstart\n", + "\n", + "nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", + "nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", + "print(\"Filled %.0f bins, nonzero bins: %1.1f %%\" % (nbins, 100*nfilled/nbins,))\n", + "print(\"Processing time: %1.2f s with %i workers (%.2f s cpu overall)\" % (dt, nworkers, dt*nworkers, ))\n", + "\n", + "# This is taken from the DM photon analysis... \n", + "# Pickle is not very fast or memory efficient, will be replaced by something better soon \n", + "# with lz4f.open(\"pods/\"+options.year+\"/\"+dataset+\".pkl.gz\", mode=\"xb\", compression_level=5) as fout: \n", + "if not outpath.endswith('/'): outpath += '/'\n", + "if not os.path.isdir(outpath): os.system(\"mkdir -p %s\"%outpath)\n", + "print('Saving output in %s...'%(outpath + outname + \".pkl.gz\"))\n", + "with gzip.open(outpath + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", + "print('Done!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d257d72", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 0a6cfe70f0660fc67f2006cb0bc2a3cd2ce67b50 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:17:56 +0000 Subject: [PATCH 04/11] Add topcoffea installation from oshadura fork repo --- topeft-coffea-casa.ipynb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 3608c1de9..9dcdd721b 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "f467b886", + "id": "9ce0043a", "metadata": {}, "outputs": [ { @@ -30,7 +30,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "781aeeb6", + "id": "99c76a58", "metadata": {}, "outputs": [], "source": [ @@ -70,14 +70,14 @@ " def setup(self, worker: Worker):\n", " os.system(f\"pip install {self._depencendies}\")\n", "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\"\n", + " \"git+https://github.com/oshadura/topcoffea.gitt@coffea-casa-analysis#egg=topcoffea\"\n", "])" ] }, { "cell_type": "code", "execution_count": 3, - "id": "0286ca63", + "id": "f381cc91", "metadata": {}, "outputs": [ { @@ -138,7 +138,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "79e4d98b", + "id": "45f143e5", "metadata": {}, "outputs": [ { @@ -269,7 +269,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "dd0c41e4", + "id": "04cb32dd", "metadata": {}, "outputs": [ { @@ -338,7 +338,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4d257d72", + "id": "bdfbcb28", "metadata": {}, "outputs": [], "source": [] From 9606d88b552f0788c2ddee9159601d7115a56e09 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:20:58 +0000 Subject: [PATCH 05/11] Fix a tiny typo in git address --- topeft-coffea-casa.ipynb | 109 ++++++--------------------------------- 1 file changed, 16 insertions(+), 93 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 9dcdd721b..7e8ce10d8 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "9ce0043a", + "execution_count": null, + "id": "d2c73461", "metadata": {}, "outputs": [ { @@ -11,14 +11,7 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n", - "Installing collected packages: topcoffea\n", - " Attempting uninstall: topcoffea\n", - " Found existing installation: topcoffea 0.0.0\n", - " Uninstalling topcoffea-0.0.0:\n", - " Successfully uninstalled topcoffea-0.0.0\n", - " Running setup.py develop for topcoffea\n", - "Successfully installed topcoffea\n" + "Obtaining file:///home/cms-jovyan/topcoffea\n" ] } ], @@ -29,8 +22,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "99c76a58", + "execution_count": null, + "id": "0b973d1b", "metadata": {}, "outputs": [], "source": [ @@ -70,24 +63,16 @@ " def setup(self, worker: Worker):\n", " os.system(f\"pip install {self._depencendies}\")\n", "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.gitt@coffea-casa-analysis#egg=topcoffea\"\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis#egg=topcoffea\"\n", "])" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "f381cc91", + "execution_count": null, + "id": "8540b700", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" - ] - } - ], + "outputs": [], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -137,48 +122,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "45f143e5", + "execution_count": null, + "id": "8bb233e8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", - ">> TTZToLLNuNu_M10\n", - " - isData? : NO\n", - " - year : 2018\n", - " - xsec : 0.252900\n", - " - histAxisName : \n", - " - options : \n", - " - tree : Events\n", - " - nEvents : 14542666\n", - " - nGenEvents : 4876491\n", - " - SumWeights : 19992000.000000\n", - " - Prefix : root://xcache//\n", - " - nFiles : 18\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" - ] - } - ], + "outputs": [], "source": [ "\n", "samplesdict = {}\n", @@ -268,34 +215,10 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "04cb32dd", + "execution_count": null, + "id": "021c1708", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[# ] | 2% Completed | 0.1s4s\r" - ] - }, - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'topcoffea.analysis'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mtstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'topcoffea.analysis'" - ] - } - ], + "outputs": [], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", @@ -338,7 +261,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bdfbcb28", + "id": "330c0d3b", "metadata": {}, "outputs": [], "source": [] From e4c83dbe7dafd51ad4735f147e1d291f7daf0961 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 13:13:13 +0000 Subject: [PATCH 06/11] Update plotting part according Zora's example --- topeft-coffea-casa.ipynb | 206 ++++++++++++++++++++++++++++++++------- 1 file changed, 170 insertions(+), 36 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 7e8ce10d8..8e660d9d1 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": null, - "id": "d2c73461", + "execution_count": 1, + "id": "a800ee7d", "metadata": {}, "outputs": [ { @@ -11,7 +11,14 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n" + "Obtaining file:///home/cms-jovyan/topcoffea\n", + "Installing collected packages: topcoffea\n", + " Attempting uninstall: topcoffea\n", + " Found existing installation: topcoffea 0.0.0\n", + " Uninstalling topcoffea-0.0.0:\n", + " Successfully uninstalled topcoffea-0.0.0\n", + " Running setup.py develop for topcoffea\n", + "Successfully installed topcoffea\n" ] } ], @@ -22,8 +29,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "0b973d1b", + "execution_count": 2, + "id": "317b25dc", "metadata": {}, "outputs": [], "source": [ @@ -52,27 +59,23 @@ "\n", "if hasattr(__builtins__,'__IPYTHON__'):\n", " import sys\n", - " sys.argv = ['']\n", - " \n", - "from dask.distributed import Client, Worker, WorkerPlugin\n", - "import os\n", - "from typing import List\n", - "class DependencyInstaller(WorkerPlugin):\n", - " def __init__(self, dependencies: List[str]):\n", - " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", - " def setup(self, worker: Worker):\n", - " os.system(f\"pip install {self._depencendies}\")\n", - "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis#egg=topcoffea\"\n", - "])" + " sys.argv = ['']" ] }, { "cell_type": "code", - "execution_count": null, - "id": "8540b700", + "execution_count": 3, + "id": "38f87aff", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -122,10 +125,48 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "8bb233e8", + "execution_count": 4, + "id": "c8bf7134", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], "source": [ "\n", "samplesdict = {}\n", @@ -215,13 +256,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "021c1708", + "execution_count": 5, + "id": "67d41c95", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "CommClosedError", + "evalue": "in : Stream is closed", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mStreamClosedError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0mframes_nbytes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mframes_nbytes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframes_nbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mStreamClosedError\u001b[0m: Stream is closed", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mCommClosedError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"tls://localhost:8786\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdependency_installer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m executor_args = {\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mregister_worker_plugin\u001b[0;34m(self, plugin, name, **kwargs)\u001b[0m\n\u001b[1;32m 4054\u001b[0m \u001b[0mplugin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4055\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4056\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4057\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4058\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 841\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 842\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 844\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 845\u001b[0m )\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 354\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_register_worker_plugin\u001b[0;34m(self, plugin, name)\u001b[0m\n\u001b[1;32m 3974\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3975\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3976\u001b[0;31m responses = await self.scheduler.register_worker_plugin(\n\u001b[0m\u001b[1;32m 3977\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3978\u001b[0m )\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv_from_rpc\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m 860\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"ConnectionPool.\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0msend_recv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 863\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv\u001b[0;34m(comm, reply, serializers, deserializers, **kwargs)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mserializers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"raise\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_finalizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0mconvert_stream_closed_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;31m# Some OSError or a another \"low-level\" exception. We do not really know what\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mconvert_stream_closed_error\u001b[0;34m(obj, exc)\u001b[0m\n\u001b[1;32m 126\u001b[0m ) from exc\n\u001b[1;32m 127\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mCommClosedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"in %s: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mCommClosedError\u001b[0m: in : Stream is closed" + ] + } + ], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\"\n", + "])\n", + "\n", "client = Client(\"tls://localhost:8786\")\n", "client.register_worker_plugin(dependency_installer)\n", "\n", @@ -247,21 +326,76 @@ "print(\"Filled %.0f bins, nonzero bins: %1.1f %%\" % (nbins, 100*nfilled/nbins,))\n", "print(\"Processing time: %1.2f s with %i workers (%.2f s cpu overall)\" % (dt, nworkers, dt*nworkers, ))\n", "\n", - "# This is taken from the DM photon analysis... \n", - "# Pickle is not very fast or memory efficient, will be replaced by something better soon \n", - "# with lz4f.open(\"pods/\"+options.year+\"/\"+dataset+\".pkl.gz\", mode=\"xb\", compression_level=5) as fout: \n", - "if not outpath.endswith('/'): outpath += '/'\n", - "if not os.path.isdir(outpath): os.system(\"mkdir -p %s\"%outpath)\n", - "print('Saving output in %s...'%(outpath + outname + \".pkl.gz\"))\n", - "with gzip.open(outpath + outname + \".pkl.gz\", \"wb\") as fout:\n", - " cloudpickle.dump(output, fout)\n", + "os.system(\"mkdir -p histos/\")\n", + "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", + "with gzip.open(\"histos/\" + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", "print('Done!')" ] }, { "cell_type": "code", "execution_count": null, - "id": "330c0d3b", + "id": "f2250b7d", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import print_function, division\n", + "from collections import defaultdict, OrderedDict\n", + "import gzip\n", + "import pickle\n", + "import json\n", + "import os\n", + "import uproot\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.hist import plot\n", + "from cycler import cycler\n", + "\n", + "from topcoffea.plotter.OutText import OutText\n", + "\n", + "\n", + "path = 'histos/plotsTopEFT.pkl.gz'\n", + "outname = 'temp.png'\n", + "\n", + "# Select variable, channel and cuts\n", + "var = 'met'\n", + "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", + "cut = 'base'\n", + "\n", + "print('Opening path: ', path)\n", + "hists = {}\n", + "with gzip.open(path) as fin:\n", + " hin = pickle.load(fin)\n", + " print(' >> looking for histograms...')\n", + " for k in hin.keys():\n", + " if k in hists: hists[k]+=hin[k]\n", + " else: hists[k]=hin[k]\n", + "\n", + "\n", + "# Create figure\n", + "fig, (ax, rax) = plt.subplots(2, 1, figsize=(14,7), gridspec_kw={\"height_ratios\": (3, 1)}, sharex=True)\n", + "plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9)\n", + "\n", + "# Select the histogram var, channel and cut\n", + "h = hists[var]\n", + "h = h.integrate('channel', channel)\n", + "h = h.integrate('cut', cut)\n", + "\n", + "# Integrate over samples\n", + "h = h.sum('sample')\n", + " \n", + "# Plot and save figure to outname\n", + "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", + "fig.savefig(outname)\n", + "print('Output histogram saved in %s'%outname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59e67150", "metadata": {}, "outputs": [], "source": [] From b9330618b316fcf42b0b8106aadf720c55f844d9 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 13:20:02 +0000 Subject: [PATCH 07/11] Remove accidentaly added cache (.gitignore?) --- .../.ipynb_checkpoints/run-checkpoint.py | 101 --------------- .../work_queue_run-checkpoint.py | 117 ------------------ .../__pycache__/__init__.cpython-38.pyc | Bin 155 -> 0 bytes .../topEFT/__pycache__/topeft.cpython-38.pyc | Bin 18648 -> 0 bytes .../__pycache__/topeftenv.cpython-38.pyc | Bin 7267 -> 0 bytes 5 files changed, 218 deletions(-) delete mode 100755 topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py delete mode 100644 topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py delete mode 100644 topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc delete mode 100644 topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc delete mode 100644 topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py deleted file mode 100755 index 41ea9282d..000000000 --- a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -import lz4.frame as lz4f -import pickle -import json -import time -import cloudpickle -import gzip -import os -from optparse import OptionParser - -import uproot -import numpy as np -from coffea import hist, processor -from coffea.util import load, save -from coffea.nanoevents import NanoAODSchema - -import topeft -from topcoffea.modules import samples -from topcoffea.modules import fileReader - -if __name__ == '__main__': - - import argparse - parser = argparse.ArgumentParser(description='You can customize your run') - parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') - parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') - parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') - parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk') - parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') - parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') - parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') - parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') - parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients') - - args = parser.parse_args() - cfgfile = args.cfgfile - dotest = args.test - nworkers = int(args.nworkers) - chunksize = int(args.chunksize) - nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks - outname = args.outname - outpath = args.outpath - treename = args.treename - do_errors = args.do_errors - - if dotest: - nchunks = 2 - chunksize = 10000 - nworkers = 1 - print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) - - ### Load samples - if cfgfile != '': - samplesdict = samples.main() - elif os.path.isfile('.samples.coffea'): - print('Using samples form .samples.coffea') - samplesdict = load('.samples.coffea') - else: - print('Execute as [path]/run.py [path]/samples.cfg') - exit() - - flist = {}; xsec = {}; sow = {}; isData = {} - for k in samplesdict.keys(): - samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) - flist[k] = samplesdict[k]['files'] - xsec[k] = samplesdict[k]['xsec'] - sow[k] = samplesdict[k]['nSumOfWeights'] - isData[k]= samplesdict[k]['isData'] - - # Check that all datasets have the same list of WCs - for i,k in enumerate(samplesdict.keys()): - if i == 0: - wc_lst = samplesdict[k]['WCnames'] - if wc_lst != samplesdict[k]['WCnames']: - raise Exception("Not all of the datasets have the same list of WCs.") - - processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) - - # Run the processor and get the output - tstart = time.time() - output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.futures_executor, executor_args={"schema": NanoAODSchema,'workers': nworkers, 'pre_workers': 1}, chunksize=chunksize, maxchunks=nchunks) - dt = time.time() - tstart - - nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) - nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) - print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) - print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) - - # This is taken from the DM photon analysis... - # Pickle is not very fast or memory efficient, will be replaced by something better soon - # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: - if not outpath.endswith('/'): outpath += '/' - if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) - print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) - with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: - cloudpickle.dump(output, fout) - print('Done!') - - - - diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py deleted file mode 100644 index 05ed75686..000000000 --- a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -import lz4.frame as lz4f -import pickle -import json -import time -import cloudpickle -import gzip -import os -from optparse import OptionParser - -import uproot -import numpy as np -from coffea import hist, processor -from coffea.util import load, save -from coffea.nanoevents import NanoAODSchema - -import topeft -from topcoffea.modules import samples -from topcoffea.modules import fileReader -import topeftenv - -import argparse -parser = argparse.ArgumentParser(description='You can customize your run') -parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') -parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') -parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') -parser.add_argument('--chunksize','-s' , default=100000 , help = 'Number of events per chunk') -parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') -parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') -parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') -parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') -parser.add_argument('--do-errors' , action='store_true', help = 'Save the w**2 coefficients') - -args = parser.parse_args() -cfgfile = args.cfgfile -dotest = args.test -nworkers = int(args.nworkers) -chunksize = int(args.chunksize) -nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks -outname = args.outname -outpath = args.outpath -treename = args.treename -do_errors = args.do_errors - -if dotest: - nchunks = 2 - chunksize = 10000 - nworkers = 1 - print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) - -### Load samples -if cfgfile != '': - samplesdict = samples.main() -elif os.path.isfile('.samples.coffea'): - print('Using samples form .samples.coffea') - samplesdict = load('.samples.coffea') -else: - print('Execute as [path]/run.py [path]/samples.cfg') - exit() - -flist = {}; xsec = {}; sow = {}; isData = {} -for k in samplesdict.keys(): - samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) - flist[k] = samplesdict[k]['files'] - xsec[k] = samplesdict[k]['xsec'] - sow[k] = samplesdict[k]['nSumOfWeights'] - isData[k]= samplesdict[k]['isData'] - -# Check that all datasets have the same list of WCs -for i,k in enumerate(samplesdict.keys()): - if i == 0: - wc_lst = samplesdict[k]['WCnames'] - if wc_lst != samplesdict[k]['WCnames']: - raise Exception("Not all of the datasets have the same list of WCs.") - -processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) - -executor_args = {#'flatten': True, #used for all executors - 'compression': 0, #used for all executors - 'cores': 2, - 'disk': 5000, #MB - 'memory': 10000, #MB - 'resource-monitor': True, - 'debug-log': 'debug.log', - 'transactions-log': 'tr.log', - 'stats-log': 'stats.log', - 'verbose': False, - 'port': [9123,9130], - 'environment-file': topeftenv.get_environment(), - 'master-name': '{}-workqueue-coffea'.format(os.environ['USER']), - 'print-stdout': True, - 'skipbadfiles': False, - 'schema': NanoAODSchema, - 'extra-input-files': ["topeft.py"] -} - -# Run the processor and get the output -tstart = time.time() -output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks) -#output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks, extra-input-files=["topeft.py"]) -dt = time.time() - tstart - -nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) -nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) -print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) -print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) - -# This is taken from the DM photon analysis... -# Pickle is not very fast or memory efficient, will be replaced by something better soon -# with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: -if not outpath.endswith('/'): outpath += '/' -if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) -print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) -with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: - cloudpickle.dump(output, fout) -print('Done!') - diff --git a/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index ca8cca7a5e9d14632ec6078762204275a04cc61c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmWIL<>g`kf|S1p6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6enx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*FCB;@KAq8}fhnU`4-AFo$Xd5gm)H$SB` NC)EyQ@@F7s002(`CL90& diff --git a/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc deleted file mode 100644 index 02a68d8510b6ab5cff39978cc1d501d3577f6c3e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18648 zcmb_kd3YPwamV5&2!a$viMmA{mPJ|CLCLplOX8u!Iw+Br?6qVF#9M*@u>iEYAVvDE zd{A~S(@|n4X%aUPlemr3#!1_xN9?9a+NNplqq#O|d?ab3HvRg2ZNGkL|7m|Sy8r=7 zw&lp&Z)a!b&6_uG-ZgLWmUlNb#eDd?`0rml_5Szyd_O0m_AdltCvNk<0fbK|d7rMR ze#ONDYJhn`HOSA98scYI4f8XiM)}#G#OeW|+ClUc?x zvHWs>NoyFB-Axpv$jSwxBI3Pb*;8a zTZ`v4LK*ONwp>KreTSTIHd81XT4%rsi|kldXig+s7)#}|!U-0~G^5k+1V^%_uFnAWK{@xvu6TP*aa3=SSN;fJjk_DS`6EzLk-nr)`T-Toe<5H6 zg(Ce{=t7u!$_!x<5g;3=?3vJ15S*xp&XH(<9V8K&3C#ppqOndQEaD>0+QOnqGp zTSeDZdwiSN{w8UC(>tFB$mxWjA`5w)YILEQTEycwFk8myS~PZ8EA z1AuYi!(4dVAvHcw)(;OK*Rq)r%XGqKN-yO#rBs9-`g2we07IH8`h zu1`22L66CnLuGwPGdhaH9om>yAZ5MoGm;)j6$)D338c$dcFeL4hLO>nm^oounx3+< z>B)ryRlMWwLRvGdRJLGEIN@}$%*wjmCoN`r$}}eztkshboY3-ly6v?c>-T8KIy)}5 zk$GQX^9opa%*lECHLF7}=dElhpG{}2iH?gDOgQ1f5Y{1cvXvfl1Spu)e_MUwT_o~fBoZFaYo(F}D@(}-2JQPF0C~J>r4!V)AaTh7 zEhVyrOb4y29j>HTk&BDTgrHt0rk9I_dNC)MtCK*DJLX7)v@M#Inw(!>)(P;|UR;4f zGPv;<{XzxV>4fz9EN;1+#R=u=)7%o>H2;V-*|c{pRoJUW_E%WGJLee-QG&6Qz$yZ3 z2&^MOPBk_V*hpX#fz1TA5+HvX+X&o1;6?&B0XU7^wr&ezMUN)kerq~S$?Q)vQ~OJ)mME14w!ndBj#5>~>&pt2g*{@1wFe~-1rl*uKR z#5F6PU6!8G=PF}M7Pm$(CIm9iIKTDB*ZX6qnIfB%Mi9&WDk zGW1QUI4+Kn1H*5LBp2g+PU?R2Wo5L?{TQ093>?VLK|L zJ-(Am>;~B&{i9(+NdE;tLOn$!AniXbP=3IU0S4_xz>pmW4BJhB5j$Z;Wjxn_JBGUv zcYG=+bz;BaQ#-vCfSPc6_&xuCfVZR@NN^^2`Sjw#GYyhXB&pYZo(>bc~y?YY$IlnJqN%75N({6?%S_(g}k zOh(QJ%rjz@Oyo8!@&P8=0LAJZz8cXbR@}GY z-hun}sVKCdMd4ZWp{keB@2g%;u$L2%rg|*oc3jE6TnS%xz(aS)mE04qMY*da($L#o z&H~}~_SG^d){pwn2aScI(_T{uiVgM}$lNK{j0Wws*5(U4p=F7|<12mUU( z3fQ$(Y<(5$gm**cQ&f_@0o>idnjw1~p569Ft4CfZH^_}s0sHz2E>?-_ri1pTD&5BP zW~~5qVmb=7q(IvKJ{9^lb!0wSfxqj5WCwHV;_g?Oa;1jpqm%C2fun2uww2b=f zKG{ch*?U;mZs@vE+~n!nEBEG(Lf78hKvma1)&)Ls=jhrm_p`3MWRL8XecW;f7~jMA zLAh7%llvJz#Q0r|ANJbh0JVvSA0!-g?DyzHHED;vv?K66)i%M9+Kg&V)RCGNJPIkQ zM}i@>G1Z5t0d8@(od;f5m)qWhudB_KyZd$Zx&7nV>*{mmj$4D`X4x-qmq&&J3-J<6 zp}BcI{M<|P68G_5=IoREs+@hyK^s>&eazV-_f$D(dFVmgRXO{abCPq|E8HJAL+jZ$KV% zh<$=0~tJSp!%|GOXULzX2NQXeA=iE{hcJ;3jA`_WX) zPF}e*&-;z`sW*^=R)$79RWldD|8wSyp_+NrO|6|r)9m@7x=}8q;88BTQC{&pxJa2Z z#^Fir4ak${huP1X)T(~gm@_PgtDIrd>6T_^WJadp`4O4PU1w+INx+=EA8=HT0OsK_ zvN*wz#xq$?lslf&z|-#dM~moJF735Q;SJ<)TcpT7N3?d?Ilfdg$3tdLd!6PMDd80+ z=5TRKCcv|}!Q9i}Vm21HdR)_!t)1^JPqKF2E`xWsEY*##G3K4FYx!~1)Zo5Nkova4 zeVd?59fL28+_9=R&mf1%z9&e&H`(_DUFtY;n7QLj1>ygRH&%w%29s~B952@cZ?G)4 z_2e5Y%gtrWGS5$C_~D)z#lr^(M?WC$en707qXAyy&cgrqvea4h^s{y2>bB7!YQS+R zkBqbK9E-8@Aabd%nhO!To6CMkS}bLu9V^yYXhqi=;_jD3SFCxsYR$u#@gANtzt&sx zaMhYes@7aC*Q_;DT#G3=RUO~wnDdByq{?}eISR!gx24okw@bV9E>n<-YAMk_To^i)$$bK3# z4(^&r@^>*qJzW=3e+Yg%0d5?z9`79N56cgOcT#?s_f5BpJNTWz1$hBKZ~7j^?oRA@ z)IUt`NoM4XwOh{Q?wL}=oukD0NDXHvb3V$x`3R!%k1_U9V9&^p%8$`5Rz5=zGvP;T zZ2Ud>kvXX!sY(5P*mhX{e%+k*apwJA-JJF;M)SvcP9sRO*vENJBS_Sy6K_H$FLME&VJk`YiVag4Ft-<-S1BrD!GlthbVV2KZ;#OVmgH^ta9HdE?K1 z>-?@a{+zu*yT)h4j!VYTU$8g8N2zaAq)(=Mf+=6pZ#@l0B^th?QdA=oVb9~A`&}!{5<(}`3<}o1YGR9 zFf0y|R^tizwL(mWWO%7heyz!O=;9+pStpgcx= z@UZ)Zw!`ucd8gdz@p?SofV^8C_juR=v)mauDW%859+-I(@&RdkJnWsBr^|vYdc3_J zFDXySl*jA$cwdlTlwb0AM?BtN%FoNc@_0u*-k-`plYj2<20Y%U~D)>;<&g++$+>f$o`J}HtKdv_Q_tXy??#a^*8a3%RQIy?!h~OeUST`8o7JrU6=4w zd7J!OS4P|?PKcA@{!GyRE|-Pe(bjE=#x^De_xY+zZ|a8-xW!5N~FY4CU}$2 z)0dRzy^#NQTlphqOr=v(Yjg%#<-(aj6JQLVb=2?sp zuMBK8^Trq>-Z*|=sF}gQIkN-5lG{I#5pg?y6?DcmS_QZ20{rg|IlAQIPJ- z)lYhS=YuCd zWdBS$7fLferXD}xw_mhhlHZZvZTHFVwYyxyepx)i>B8=o|G?NW`xW^!`{(j!Vrn!D z+rErBfT*)<=}V}|>B>B7|DteAzAV4Xa=(!;BYI!)D8=Z%EGz^6JK!%jTA07w;|Jx- z_AfCXFE`d#>Lh=Tc_fax_?J{eqU>MDmotidC4;xJvYVj&Px9Njd*wep;j{l){zCpU zB*_oI!Y@R-Btb)*!>=zFy*X;R@euq0DW1Xp1)iaqYq{}W`K>DVU*(IUh&nAd{tFg9 zN_8-wgVo<-t3Q8!+3DvBt(XlD%9rHJ@G!K&vR}jVMfUu^$bZGbmzU)$vRnRI{t6y? zY`OvK2l7GlX=q+ykax%nta(luD~t}3y=)1mU!{b%R7-e??142eicv1nZ>{EfQRGZKSRinj%mJZ`qXn8lZbQp;%*7782DSQ=eYqE~Bo+TZ|(JR*R6zg~lI?Al$ zOI17Gd&N55!#Z|D#~5@hGoJJO+F>|XtmC~Ek08g#aDM>zIoxCNr#LS5zC%t#q^y*w z;h+qiA>MT+YaYU38z(rSr3@!{#?;bIQ=$GyS=Q`99yKLgLIUQ zlJ^vg;^P0|Z;;MBr(mlS9o4jw$m-_BW@O8J;QjRX?46Wf7|?*T(q?yFOBIZ@v*|y+ z)1cE>7XtulxMU>@Mcwh2EMrlHVWh^@D4j}65=J5!t>6TVks=PS#PX@hiDWvJ9>LLS z<2dqUUyh*gpZw3++zZzC?#%pR>OUn+MQ7*PA#L?j6hPxKGnIc4&He zBM!ak<&zz^cieYPc4LQopM<_HWbDd7KmOM4yodwPzCIL7mOBk8Gp!Zq(AvdD&=;GC z>DRU)3vNxr6TZ=OC!m1{=%9cB=;$ROJ{agUz&Y-LLmb@-r1E*A2rWj5z-a;o0h545 zpbUT$bEJ@O^$iV=+RkpavF+cqt7oMoCdvyQfAHRPU_qf>CRC+IqxjE9Kv zFo8!1Oc6Lo;86mP5qJxMw-R_8fwvQQ2Z6^4tO4kZUa>eI*W;6Vd_Iql>+vByKC{Os z^;SWn6Tx7~;%u85;YPW;q&6_pQyeciQMbXsjJX(b)o7vU9y>M8kYRLourp*VsNgtT z$%j&%20E9Qrbcv{2*t5HmqcfsCg~^KoagaSL;HJ=qMQgGTnOj^mmVM*LntJyiIUbC zbREM!VLy?JIvb29N#j!l*plUh(fLVZB?0oD@ooYg0Iw^~-n+awx9sJLxV$)@oxF1K zb!6g|i?4rOaf43$b;fGk!$rJ)#?FX)7WB%E;ZF71;-Q+}&n?LPo92MJCgFbjdYT@h zy7P$Qk-~j}rj$BO-0xpoQ*GDbv2x|&H;@TeF3v;mwZ*Xn;#hNvR&nDNXm>&;jz!}L zEY3MQ(dr@_q=ncC45rFXV83RmF?X@$#XBI*OsIY_sfNsKX`8^nE*4E~pHLgUwU?xf z6Qn$VqSQob_^6iOXBM|?OJcHD!}?fiaB#1h&^3`Q>-Y#qFIkk`qJ_KLJsR#$jXGf* z64z5_jJwGTg8&sX$V{~b(@M6#&sEKeqxqs~?agA=tXRT=36$4|g16putP57C(R8se zc1*L1Q~@oksezHArZ#E&3S$Seg|gWX#89cY2DjGmf;taDffyX^w+`bIDlqyjHJTN> z3Rd=D8FGxFG&PI>fC`zH6+K!yHF2;gv_sl>&q31(6w1028A+Kb%QDmk2=?(uQCPwd zIXF%9Ar+1-r%alLoQC0SK95MFps9^GA&=lea8Thdl$^*gzSgp~bvZ3V20q(LBTF)6 z7^w+UT{xUQqlrC6RvaBE_8|ew$F6)H$LJ~kFpU;!me5c+n-}Cwj+M~BP9%e-BD+G! zfW`Uf6IBTS}8m~1L^7YnA9waUD925_WbZJ-&v2jSIj#9jq>2}k{5 zJjY(!^Cr%)DdCjurum$QrlWz;ESV+n0HXk;@P%~0c( zi}V$Z?Rn!?q|@gMbi7+lRL*xNtzt26IuSPjLa#^(;Y6)s61}4|VeBH!J4tgQoyw<^ z<6Z8OW*QW38T$bobUe5nUo!B5;{>UvIH4@Qq%iS`A6)JPP~Fad#AI7s;q z5jaespFlf-qX16Si|b8C5pINWir9V~6M%(TK*^~w8UhIXl6qN<=;dSy(KjvTE)!i} zb6KGR54%u-XNzEghEK#AFCNfJkhew*O=FaJEJ+B@8QgZ%Y-$5~VacL%{A%2_-wmMQ z6&&Bkt^uO~BBmP6Ajn4tHr-%|7Ei~Y+oEzPIF7g?YwkwBBnh<%o0TGeeaIPjiFyzj zjCWCyUM$j60`(*>7lx9h z(2oaYt{P0NxN9NpLqJu`M@o(PZek6Cl;JB31d5oHsqUTxv!Xf{pCqDkR0^vEV_{9> zXQTII`x7p4xKc$_m#WBuF^prVX55Wd8TG7%aRO8;4muA-ilpCv1+YZd)+YdP5JeJtJsWu~QL&rDS zcJxcNrH+A_a#n=uR42j=5{XwcLBxrUr40TYPmR*Yo(J&KPz{b0%a+ZqR+n7iHeRMxbcS?4kF5{+N=eUM)&giTYXs_zKUxf=iSel{WuEB?h@zV1v@9 zv?|Lyx+T!1bSQB*4KNNI`IUvr!ay5zwBcHzEK*Ryf=z)Ar3F}vvd|y%g_R(1@B;*6 zRF*FeZXD??u!gim{p%IJ{H-n>SpjV=&>2+{{vhxUr6T~Izr#-@EklmVlPsZY8Ejpr ztaJ4!>;0{PR@h&?qNK6SzeU+XJlG%gZ&JFHF4o!x?kwN)M@>3v$u&foe&m6l+-nP- z>!A~{Qd(fGver*^BKs;38Ur2=HEbmxMB{8nD{Cem`J8P;zMV>s(&Hy;p|S(i&P(t` z@HI=hFfiM$gtde=27+ArYGo07imaJ)1y?Am(Q=6@zXe{QHo@ihp1>01S@$9?=r)#t z1+%51jbbQ|TGFq)%JrYImN_ZEiN)ML;lftfj9%jpqyM#_HDlx<@{2zRoYMj^UqYdq zz8j@9vY7f{Old=(YE{;Px55`B46P9Jfv()>Lz_T#kVZ(v{3JUUTV3T{4%@CwUgn!j zZ=BVAlj+BBQ*d|VHtFTF#J@M;FF|>j;^p3MFnKF9cic9X@CLxsnSmbnYf$BKU!vAX z9_nnMk8Ek0N+$7zZ&ELca-Q&bGI_e3%DXAX9u(_Dp&tLH1e-6~ix~$9e4oG%2=vy2 zJ|f!ymf$w&MJ5L2azDB&?qKvmpL=a!8L~mgNAJnxWMb21ci-5DIMm!cnee_a-sEyS z*EtRO$?cnl4f^dRaSvZ-0&FQM?m!x0%ln9!ZUl zj;9P^e2AzpwmW!(%AZBM-%VrhOMBN%WhD1yvvp6{f8O% zlkfRtf4sl)Z(nxp?q<&$KctF8uqu^O*q>6z!(P4!zj^;4=W^}pM0dD#x`R5bxdp(T e*Lf^sJi9Xi=2wg+%(_ZTq_xuEgHeq7xbpv{0XPK! diff --git a/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc deleted file mode 100644 index 92ffd4d8773d148881fbe15f61db0e2edb605323..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7267 zcmbVRON<-Id9LaQ*=){(Lye@7G#a(2hnE|1^(J8Rf$Z-696g98Ks6c{<=;Dc`oa>+Fa5};23a?&}61e1sJ{ng|Q zXJk7_D0NkJbv^(3|L_0)nh$4ZZH8a-AOHH}f4acfztGF{&%n#uc*6fiBADQ9R!wtW znfosytK8s61QEs@$qtD$iAODz~e)XN~fEtUBXatyw&CLKFID z8fU_&&I_|z@Yra7CKPenBE9Rc_>M1cV3aB|P7DN&G zk~kq2k)IJK#VO=x#c8pG{G7Nb&WUC8Sbm3z^J3*W6D!{NUA?;ET@V*Gwa=N{_Z)9| zm+f+_gnz)wrI(NaTN_-t@|qjgBfsO7!<$!LYkQBqcGn%hb-XZioA^@J5>tIjGL1lX z+^A}J-N)hJ=k`3p-)+le-U?(H{L z-&%d6bj>NHogQ6NgHn6?rL`4I-1O3Js#&Gy);Fn>@Y`+5w`CASA|0mD@Z8d)gJFiM zG)%`W^+Qh{S=o4AtmvS})EY`XzlY{i-_p_Sp~jzktNI#6PZyi z*7q>V9_K8wW22R)D8(w|07WAWDKT0wmJiPb2A3jWqcinc|6Yr;u)aXgJ9WW|&x*qNYmR zd%@&6EOP%#*EfQWcfH;TE3M%1j@!K+1-*JeI&t0Yy6v6N52@<@gNKxPjR+I0_I8qD zP4>GrB`c(An%6L3IERGs6;0!dyvR>+o6Ga4sgp=1KPqSO+{F`KLlUtO7ks1%ZKTKA z9yDMNDbCA?nb3t1^M(;?Kh}OoXJoDoF8oe6jIbNX1cG-UjezpUzU!zJ?el9%)=%5) zYn3_#lObVxGBJg>-fyZCk_(txt*)#m7WP%^xzPrsw&8ZWUOO>*o6S&(DNUBqJULw( zk2~HwSr+U*yg_>@B4K=qYm{c1<7@i6px=C`fOs?@NykK3t8qqrv%KE+klKqT~D$E!>a} z<{x|sRgj~SsfKGz=G!Fkavm z)NL$xrZMfm(p_g+{$6p>}RCObik+?**MnEReNiyeLgmJ;c8TU4)qGV z2(MyY*t@iLYMoLj zdU~`ZinRAJidxUC35rH%MrUdDSn+sI9i-R$_Mvm# zw3W+l3yrEnUyKQhQ=6_n11ASLWMU<{bimS%}_E+34yaDkdh}TA=8m3k-)EQ zz>l?)jO;|x^UAqob~0#f!wol*+{8}9#KghWz3S}2>5;Lg174;9uTWB<_TkgrUeANW&V+%CJW)2P77bdH?i09u$Qhm8nGc8%8ywPBEXiwK|rLjc8hf_0F^3z#I`KJP8M|T zF9134Y=dRhEB9)zG3=PnOoE*)fX|i{(~;O~iQe?1K~8#)!Fgo}LA`tbEBDGg(d$o0 ztdSe_Q&vYx+;;u$V6jr^cOwZK@d{3fSV^Q`NGrT7Y0}E z2K}~hxFD3l+lZt1`&Il+2nD%7~zlZ7EAR zqG@CrWU7Qg%6&IskdJ1D{8JY5GwhGFE#reJ+<^1YR==3=-U}|ji7r#rz`jNF8lvgl z`HTij0YGDeOC$;KYi}%()mzFa!aFETddS~HmxM_&f|m-V%_K`;M_dMg=-znqHxxw# zl$Jg+A`{#YBBn42Q7bHPs2%%ms_&VD?>Ho|MFnwywyQ(fV$3(Q z^1UcC$_fNa2ZTjKzyv@9VR#P~`HT8|@+w_5{^yuOHbM$(Xc>bXkQ6iVs0k zT3Q~JFZkLEJ<$T78(C;#fgp4!(1t(fjpHuFy47|&>%zVBCm8Kt@vy;4t={%r#3SV9 zrw==ANcO+!?F{TjA0ZI=-~ZF{WE*qH4kcTZ9EDvOqV7+rK0-3M^icZ101ynd81V{i z01GMG@_n>TjAlDnmtE>hl1T%STU1i;a6p-X8eS)O>>-Y^)u^@5AHW5VQAg3{u$r3) zAH~T=Ln&#vqbp8krp5KgXfS5c4tL;WEx-f7LFJMFTcEuRK)Ile(=r9xj#b zG>k&TqoPKNp`8kN>Ez^UAi?VhY==&z?>rq7MZ8qTGmlrI?j}&Hn3z+#JXuA_#%d=a zo7yJ6I12h4TAGdU4fFl~S3$!~#!ok^?spJB7mjZs$&Qz=RC|y+5a<*wE*#f$3;eZ5 zJ~%0^L8{^*UIXdMod;BBK9L}}AfW9&;#N{Z#cdOVPT*HF+_EWSYCoie$Z1s_o%HH{ z&-*3LbiC0+XS#=eV`o|bJ+*R(Gnw{pmQ9ZJEaFdGcI=`}k{!n-FA4LGSPyY=TDWqp ze*;}hDeAEh6j48+t#YLnNGPsA6iwBm+%9Z^F0^*}_f&}WDceOLNP!}-o$cxf3CBS# zuDZ<89O{)fXVogpDtM9xcSyF8?-N z&gH44;6oM_M<>M0Xim(I77<3`J~p<96cDo`i;S84chzPdv*WUj>J;3g+&YPXxNr~< zqh@q!k0H!-&;xh3Q$3cjMhb}$Q0}oG{a^_H1Rqu$+R-_zKNs7ygU|Was_I!h?1=z( zxpkiIf8(4u8L4p);4ZgT#HpzOce!;z1-O3+bjd4papHXIqC%I_=#n@+g)aGL`AL9# zh%V{Kpf^iv59M{7sbUh+LuoKK!C!>7_%CnJXW|=2yy6ieM4Ukpp~VRi5)2}T!%hiN znX3U(D6gUM3tr8W;it@SOh0H8Sx54>ctYaz8R+N?+!9zSdA`Zz;IxyRFrvh0xPDtc zN3X$~Uq1}3r@A@oUfso|Fs_DPCB_J^wQJ5RTO^O69RCsuYKw&-R~iA`XlYY?547qa zPL|(BYe^eD=DpPR{toqrbnoP``+)Q^K)ykp-=yR%N`8})o0NQ+5?WRoonM}sRe9gr z^r=Bf6UkI8+zNxPq_9f*lu!_*=(D17Dj+=!|fnM{uZ@- zn-b;nRcQYVuSxL$0Po>$uO0<*MtjumYNp*;An z`e%S|q``hc1^YH}j;S#JDwp5KIHb=mXLF)yn)qS%08R zM1BVi$`^4dPfDS*rHmSRmlEZi7OAAp(I7G_=jr`1B}Dm>97Oh>x|LS?kkEaYU%wl4 z8-6n}@7@3H58shW9Y3YkW;<|2D8EA8=|+9(mgil1J*JDq+fr2_3Cb!Zv^DtyO4J?D zAt^lAePZ28MRkXEAClRmCv_~v8RDA8FbcK~@0YW3_GN3qDxx%NowQ#s=r;c!#u58% From cb93581d3e73e4cf0b9acd6870ee185aeee9bd88 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 15:24:08 +0000 Subject: [PATCH 08/11] Add oshadura repository from which everything works --- topeft-coffea-casa.ipynb | 62 ++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 8e660d9d1..d3ce83902 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "a800ee7d", + "id": "5d03d87f", "metadata": {}, "outputs": [ { @@ -11,26 +11,26 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n", - "Installing collected packages: topcoffea\n", - " Attempting uninstall: topcoffea\n", - " Found existing installation: topcoffea 0.0.0\n", - " Uninstalling topcoffea-0.0.0:\n", - " Successfully uninstalled topcoffea-0.0.0\n", - " Running setup.py develop for topcoffea\n", - "Successfully installed topcoffea\n" + "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-9zf5jcvu\n", + "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /home/cms-jovyan/topcoffea\n", + "Building wheels for collected packages: topcoffea\n", + " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4512948 sha256=e020a6dde8da5bedb5658fcf58b182244cc65fc5593209bae50ba8da1f429d85\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-b7hy8t1i/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", + "Successfully built topcoffea\n" ] } ], "source": [ - "#!pip install git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\n", - "! pip install -e ." + "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + "#! pip install -e ." ] }, { "cell_type": "code", "execution_count": 2, - "id": "317b25dc", + "id": "502972cd", "metadata": {}, "outputs": [], "source": [ @@ -65,7 +65,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "38f87aff", + "id": "dd76877f", "metadata": {}, "outputs": [ { @@ -126,7 +126,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "c8bf7134", + "id": "402ee04d", "metadata": {}, "outputs": [ { @@ -256,33 +256,15 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "67d41c95", + "execution_count": null, + "id": "55f6755e", "metadata": {}, "outputs": [ { - "ename": "CommClosedError", - "evalue": "in : Stream is closed", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mStreamClosedError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0mframes_nbytes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mframes_nbytes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframes_nbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mStreamClosedError\u001b[0m: Stream is closed", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[0;31mCommClosedError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"tls://localhost:8786\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdependency_installer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m executor_args = {\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mregister_worker_plugin\u001b[0;34m(self, plugin, name, **kwargs)\u001b[0m\n\u001b[1;32m 4054\u001b[0m \u001b[0mplugin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4055\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4056\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4057\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4058\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 841\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 842\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 844\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 845\u001b[0m )\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 354\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_register_worker_plugin\u001b[0;34m(self, plugin, name)\u001b[0m\n\u001b[1;32m 3974\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3975\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3976\u001b[0;31m responses = await self.scheduler.register_worker_plugin(\n\u001b[0m\u001b[1;32m 3977\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3978\u001b[0m )\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv_from_rpc\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m 860\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"ConnectionPool.\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0msend_recv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 863\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv\u001b[0;34m(comm, reply, serializers, deserializers, **kwargs)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mserializers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"raise\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_finalizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0mconvert_stream_closed_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;31m# Some OSError or a another \"low-level\" exception. We do not really know what\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mconvert_stream_closed_error\u001b[0;34m(obj, exc)\u001b[0m\n\u001b[1;32m 126\u001b[0m ) from exc\n\u001b[1;32m 127\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mCommClosedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"in %s: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mCommClosedError\u001b[0m: in : Stream is closed" + "name": "stdout", + "output_type": "stream", + "text": [ + "[################################# ] | 84% Completed | 1min 35.2s" ] } ], @@ -336,7 +318,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f2250b7d", + "id": "53c45688", "metadata": {}, "outputs": [], "source": [ @@ -395,7 +377,7 @@ { "cell_type": "code", "execution_count": null, - "id": "59e67150", + "id": "01fb4952", "metadata": {}, "outputs": [], "source": [] From 19057882af2d1ebecfaec2e82d669ea85042fda9 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Sun, 20 Jun 2021 13:57:54 +0200 Subject: [PATCH 09/11] Add working notebook (except of plotting part) --- notebooks/topeft-coffea-casa.ipynb | 325 +++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 notebooks/topeft-coffea-casa.ipynb diff --git a/notebooks/topeft-coffea-casa.ipynb b/notebooks/topeft-coffea-casa.ipynb new file mode 100644 index 000000000..cc4380d0c --- /dev/null +++ b/notebooks/topeft-coffea-casa.ipynb @@ -0,0 +1,325 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + "! pip install awkward==1.3.0\n", + "#! pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import lz4.frame as lz4f\n", + "import pickle\n", + "import json\n", + "import time\n", + "import cloudpickle\n", + "import gzip\n", + "import os\n", + "from optparse import OptionParser\n", + "\n", + "import uproot\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.util import load, save\n", + "from coffea.nanoevents import NanoAODSchema\n", + "\n", + "from topcoffea.modules import samples\n", + "from topcoffea.modules import fileReader\n", + "\n", + "#FIXME: analysis is not installed anywhere (should be installed as well)\n", + "import topcoffea.analysis.topEFT.topeft\n", + "\n", + "import importlib.resources\n", + "\n", + "if hasattr(__builtins__,'__IPYTHON__'):\n", + " import sys\n", + " sys.argv = ['']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import argparse\n", + "parser = argparse.ArgumentParser(description='You can customize your run')\n", + "parser.add_argument('jsonFiles' , nargs='?', help = 'Json file(s) containing files and metadata')\n", + "parser.add_argument('--prefix', '-r' , nargs='?', help = 'Prefix or redirector to look for the files')\n", + "parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks')\n", + "parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis')\n", + "#parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers')\n", + "parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk')\n", + "parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks')\n", + "parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms')\n", + "parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory')\n", + "parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files')\n", + "parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients')\n", + "\n", + "args = parser.parse_args()\n", + "\n", + "if args.jsonFiles is not None:\n", + " jsonFiles = args.jsonFiles\n", + " print('jsonFiles {}'.format(args.jsonFiles))\n", + "else:\n", + " with importlib.resources.path(\"topcoffea.json\", \"TTZToLLNuNu_M10.json\") as path:\n", + " jsonFiles = str(path)\n", + " print('jsonFile was selected for UNL {}'.format(jsonFiles))\n", + " \n", + "if args.prefix is not None:\n", + " prefix = args.prefix\n", + "else:\n", + " prefix = \"root://xcache//\"\n", + "\n", + "dotest = args.test\n", + "#nworkers = int(args.nworkers)\n", + "chunksize = int(args.chunksize)\n", + "nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks\n", + "outname = args.outname\n", + "outpath = args.outpath\n", + "pretend = args.pretend\n", + "treename = args.treename\n", + "do_errors = args.do_errors\n", + "\n", + "if dotest:\n", + " nchunks = 2\n", + " chunksize = 10000\n", + " nworkers = 1\n", + " print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "samplesdict = {}\n", + "allInputFiles = []\n", + "\n", + "def LoadJsonToSampleName(jsonFile, prefix):\n", + " sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:]\n", + " if sampleName.endswith('.json'): sampleName = sampleName[:-5]\n", + " with open(jsonFile) as jf:\n", + " samplesdict[sampleName] = json.load(jf)\n", + " samplesdict[sampleName]['redirector'] = prefix\n", + "\n", + "if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',')\n", + "elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles]\n", + "\n", + "for jsonFile in jsonFiles:\n", + " if os.path.isdir(jsonFile):\n", + " if not jsonFile.endswith('/'): jsonFile+='/'\n", + " for f in os.path.listdir(jsonFile):\n", + " if f.endswith('.json'): allInputFiles.append(jsonFile+f)\n", + " else:\n", + " allInputFiles.append(jsonFile)\n", + "\n", + "print(allInputFiles)\n", + "# Read from cfg files\n", + "for f in allInputFiles:\n", + " if not os.path.isfile(f):\n", + " print('[WARNING] Input file \"%s% not found!'%f)\n", + " continue\n", + " # This input file is a json file, not a cfg\n", + " if f.endswith('.json'): \n", + " LoadJsonToSampleName(f, prefix)\n", + " # Open cfg files\n", + " else:\n", + " with open(f) as fin:\n", + " print(' >> Reading json from cfg file...')\n", + " lines = fin.readlines()\n", + " for l in lines:\n", + " if '#' in l: l=l[:l.find('#')]\n", + " l = l.replace(' ', '').replace('\\n', '')\n", + " if l == '': continue\n", + " if ',' in l:\n", + " l = l.split(',')\n", + " for nl in l:\n", + " if not os.path.isfile(l): prefix = nl\n", + " else: LoadJsonToSampleName(nl, prefix)\n", + " else:\n", + " if not os.path.isfile(l): prefix = l\n", + " else: LoadJsonToSampleName(l, prefix)\n", + "\n", + "flist = {};\n", + "for sname in samplesdict.keys():\n", + " redirector = samplesdict[sname]['redirector']\n", + " flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']]\n", + " samplesdict[sname]['year'] = int(samplesdict[sname]['year'])\n", + " samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec'])\n", + " samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents'])\n", + " samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents'])\n", + " samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights'])\n", + "\n", + " # Print file info\n", + " print('>> '+sname)\n", + " print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO'))\n", + " print(' - year : %i' %samplesdict[sname]['year'])\n", + " print(' - xsec : %f' %samplesdict[sname]['xsec'])\n", + " print(' - histAxisName : %s' %samplesdict[sname]['histAxisName'])\n", + " print(' - options : %s' %samplesdict[sname]['options'])\n", + " print(' - tree : %s' %samplesdict[sname]['treeName'])\n", + " print(' - nEvents : %i' %samplesdict[sname]['nEvents'])\n", + " print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents'])\n", + " print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights'])\n", + " print(' - Prefix : %s' %samplesdict[sname]['redirector'])\n", + " print(' - nFiles : %i' %len(samplesdict[sname]['files']))\n", + " for fname in samplesdict[sname]['files']: print(' %s'%fname)\n", + "\n", + "if pretend: \n", + " print('pretending...')\n", + " exit() \n", + "\n", + "# Check that all datasets have the same list of WCs\n", + "for i,k in enumerate(samplesdict.keys()):\n", + " if i == 0:\n", + " wc_lst = samplesdict[k]['WCnames']\n", + " if wc_lst != samplesdict[k]['WCnames']:\n", + " raise Exception(\"Not all of the datasets have the same list of WCs.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", + "\n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\",\"awkward==1.3.0\"\n", + "])\n", + "\n", + "client = Client(\"tls://localhost:8786\")\n", + "client.register_worker_plugin(dependency_installer)\n", + "\n", + "executor_args = {\n", + " 'schema': NanoAODSchema,\n", + " 'client': client,\n", + " 'savemetrics': True\n", + "}\n", + "\n", + "\n", + "# Run the processor and get the output \n", + "tic = time.time()\n", + "output = processor.run_uproot_job(flist,\n", + " treename=treename,\n", + " processor_instance=processor_instance,\n", + " executor=processor.dask_executor,\n", + " executor_args=executor_args,\n", + " chunksize=chunksize,\n", + " maxchunks=nchunks\n", + " )\n", + "toc = time.time()\n", + "\n", + "print(\"Dask client:\", client)\n", + "print(\"Total time: %.0f\" % (toc - tic))\n", + "print(\"Events / s / thread: {:,.0f}\".format(output[1]['entries'] / output[1]['processtime']))\n", + "print(\"Events / s: {:,.0f}\".format(output[1]['entries'] / (toc - tic)))\n", + "\n", + "os.system(\"mkdir -p histos/\")\n", + "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", + "with gzip.open(\"histos/\" + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", + "print('Done!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import print_function, division\n", + "from collections import defaultdict, OrderedDict\n", + "import gzip\n", + "import pickle\n", + "import json\n", + "import os\n", + "import uproot\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.hist import plot\n", + "from cycler import cycler\n", + "\n", + "from topcoffea.plotter.OutText import OutText\n", + "\n", + "\n", + "path = 'histos/plotsTopEFT.pkl.gz'\n", + "outname = 'temp.png'\n", + "\n", + "# Select variable, channel and cuts\n", + "var = 'met'\n", + "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", + "cut = 'base'\n", + "\n", + "print('Opening path: ', path)\n", + "hists = {}\n", + "with gzip.open(path) as fin:\n", + " hin = pickle.load(fin)\n", + " print(' >> looking for histograms...')\n", + " for k in hin.keys():\n", + " if k in hists: hists[k]+=hin[k]\n", + " else: hists[k]=hin[k]\n", + "\n", + "\n", + "# Create figure\n", + "fig, (ax, rax) = plt.subplots(2, 1, figsize=(14,7), gridspec_kw={\"height_ratios\": (3, 1)}, sharex=True)\n", + "plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9)\n", + "\n", + "# Select the histogram var, channel and cut\n", + "h = hists[var]\n", + "h = h.integrate('channel', channel)\n", + "h = h.integrate('cut', cut)\n", + "\n", + "# Integrate over samples\n", + "h = h.sum('sample')\n", + " \n", + "# Plot and save figure to outname\n", + "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", + "fig.savefig(outname)\n", + "print('Output histogram saved in %s'%outname)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file From 5f94fbcd0996396d93c631b17e87756b904756d4 Mon Sep 17 00:00:00 2001 From: Furong Yan Date: Sun, 20 Jun 2021 17:22:17 +0000 Subject: [PATCH 10/11] Fix the topeft-coffea-casa.ipynb Fix the plotting on the last cell of topeft-coffea-casa.ipynb --- notebooks/topeft-coffea-casa.ipynb | 154 +++++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 17 deletions(-) diff --git a/notebooks/topeft-coffea-casa.ipynb b/notebooks/topeft-coffea-casa.ipynb index cc4380d0c..3a60f14d7 100644 --- a/notebooks/topeft-coffea-casa.ipynb +++ b/notebooks/topeft-coffea-casa.ipynb @@ -2,9 +2,37 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-k7xxsyh6\n", + "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /opt/conda/lib/python3.8/site-packages\n", + "Building wheels for collected packages: topcoffea\n", + " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4514059 sha256=0ef291425ef9276b5bcb9980c1d0030fa94cedda3e13c6a83879d4a683055742\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-9jixxi63/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", + "Successfully built topcoffea\n", + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Collecting awkward==1.3.0\n", + " Downloading awkward-1.3.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (8.0 MB)\n", + "\u001b[K |████████████████████████████████| 8.0 MB 3.6 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (57.0.0)\n", + "Requirement already satisfied: numpy>=1.13.1 in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (1.20.3)\n", + "Installing collected packages: awkward\n", + " Attempting uninstall: awkward\n", + " Found existing installation: awkward 1.2.0\n", + " Uninstalling awkward-1.2.0:\n", + " Successfully uninstalled awkward-1.2.0\n", + "Successfully installed awkward-1.3.0\n" + ] + } + ], "source": [ "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", "! pip install awkward==1.3.0\n", @@ -13,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -47,9 +75,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /opt/conda/lib/python3.8/site-packages/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -99,9 +135,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/opt/conda/lib/python3.8/site-packages/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], "source": [ "samplesdict = {}\n", "allInputFiles = []\n", @@ -190,9 +264,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dask client: \n", + "Total time: 444\n", + "Events / s / thread: 5,895\n", + "Events / s: 32,725\n", + "Saving output in histos/plotsTopEFT.pkl.gz...\n", + "Done!\n" + ] + } + ], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", @@ -244,9 +331,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Opening path: histos/plotsTopEFT.pkl.gz\n", + " >> looking for histograms...\n", + "Output histogram saved in temp.png\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3wAAAGyCAYAAAC/acbPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxK0lEQVR4nO3df7zdVX3n+9fbBIi1UgIELiVQYidtBy1NyTFQtQ6K0MBQg1N1QK8g9d6Ulox667SGO+Po3Nu5D2bstAxTLkxso2CrKVO1ZFoUhBa9dkST6BEISslQhAN5kEAz+IMqhn7uH/sbujmcHztk7+x9vuf1fDz2Y+/v+q619vqy3I+Tt+v7I1WFJEmSJKl9XjDsAUiSJEmSBsPAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FILhz2AUXX00UfXSSedNOxhSJIkSZrHtm3b9lhVLXm+7Q180zjppJPYunXrsIchSZIkaR5L8s0Dae8pnZIkSZLUUgY+SZIkSWopA58kSZIktZTX8EmSJEkaKT/4wQ+YmJjge9/73rCHctAsWrSIpUuXcsghh/S1XwOfJEmSpJEyMTHBi1/8Yk466SSSDHs4A1dVPP7440xMTLBs2bK+9u0pnZIkSZJGyve+9z2OOuqoeRH2AJJw1FFHDWRFc6CBL8nqJPcm2ZFk/RT7k+SqZv+dSU6drW2SI5N8Nsl9zfvipnxVkvHm9bUkb+hqc3vT1779xwzyuCVJkiQdmPkS9vYZ1PEOLPAlWQBcDZwDnAxcmOTkSdXOAZY3r7XANT20XQ/cVlXLgduabYC7gbGqWgGsBv5Lku5TVt9aVSua166+HqwkSZKkVknCe97znme2f/u3f5sPfOADwxvQ8zTIFb5VwI6qur+qngI2AWsm1VkDXF8ddwBHJDlulrZrgOuaz9cB5wNU1ZNVtbcpXwTUgI5LkiRJUssddthhfPKTn+Sxxx4b9lAOyCAD3/HAQ13bE01ZL3VmantsVe0EaN6fOT0zyWlJtgN3AZd2BUCADzenc74v06yXJlmbZGuSrbt37+71OCVJkiS1zMKFC1m7di2/+7u/+5x93/zmNznzzDM55ZRTOPPMM3nwwQcBePvb38473/lOXvGKV/CSl7yEP/mTP3mmzQc/+EFe/vKXc8opp/D+97//4B3HAPueKlRNXnWbrk4vbZ9boepLwEuT/GPguiSfrqrv0Tmd8+EkLwY+AbwNuH6K9huADQBjY2OuED5f294Ne8b73+/iFbDyyv73K0mSpNE1iH9b9vjvyssuu4xTTjmF3/zN33xW+bp167jooou4+OKL2bhxI+985zv50z/9UwB27tzJF77wBb7xjW/w+te/nje+8Y3ccsst3HfffXz5y1+mqnj961/P5z//eV796lf397imMMgVvgnghK7tpcAjPdaZqe2jzWmfNO/PuR6vqr4OfBd4WbP9cPP+beBjdE4Z1aDsGe//j3IQfUqSJEkzOPzww7nooou46qqrnlX+xS9+kbe85S0AvO1tb+MLX/jCM/vOP/98XvCCF3DyySfz6KOPAnDLLbdwyy238LM/+7OceuqpfOMb3+C+++47KMcwyBW+LcDyJMuAh4ELgLdMqrMZWJdkE3Aa8ERV7Uyye4a2m4GLgSua9xsBmroPVdXeJD8G/CTwQHPjliOq6rEkhwDnAbcO7KjVsXgFvO72/vV36xn960uSJElzx5DP8Hr3u9/NqaeeyiWXXDJtne4rxg477LBnPlfVM++XX345v/IrvzK4gU5jYCt8zfVz64Cbga8DN1TV9iSXJrm0qXYTcD+wA/gQ8GsztW3aXAGcleQ+4KxmG+BVwNeSjAOfAn6tqh4DDgNuTnInME4nQH5oUMctSZIkqT2OPPJI3vzmN/MHf/AHz5S94hWvYNOmTQD80R/9Ea961atm7OMXfuEX2LhxI9/5zncAePjhh9m16+A8OGCQK3xU1U10Ql132bVdnwu4rNe2TfnjwJlTlH8U+OgU5d8FVu7v2CVJkiQJ4D3veQ+/93u/98z2VVddxS//8i/zwQ9+kCVLlvDhD394xvZnn302X//61/m5n/s5AH74h3+YP/zDP+SYYwb/ePCBBj5JkiRJmov2rcYBHHvssTz55JPPbJ900kn8xV/8xXPafOQjH5m2j3e96128613v6v9AZzHIm7ZIkiRJkobIwCdJkiRJLWXgkyRJkqSW8ho+9f9hlnvGO49lkCRJkp6nqnrW4w7abt8jHPrNFT71/6Hmi1cY+CRJkvS8LVq0iMcff3xgIWjUVBWPP/44ixYt6nvfrvCpo98PSpckSZKep6VLlzIxMcHu3buHPZSDZtGiRSxdurTv/Rr4JEmSJI2UQw45hGXLlg17GK1g4NPcsWccbj2jf/0tXgErr+xff5IkSdKIMfBpbuj3NYH9vGZRkiRJGlEGPs0N/V6J6+dKoSRJkjSivEunJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqqYXDHoA0NHvG4dYz+tff4hWw8sr+9SdJkiQdoIGu8CVZneTeJDuSrJ9if5Jc1ey/M8mps7VNcmSSzya5r3lf3JSvSjLevL6W5A1dbVYmuavp66okGeRxaw5YvKLz6pc9452XJEmSNEIGtsKXZAFwNXAWMAFsSbK5qu7pqnYOsLx5nQZcA5w2S9v1wG1VdUUTBNcD7wXuBsaqam+S44CvJflvVbW36XctcAdwE7Aa+PSgjl1zQL9X4vq5UihJkiT1ySBX+FYBO6rq/qp6CtgErJlUZw1wfXXcARzRhLWZ2q4Brms+XwecD1BVTzbhDmARUABNf4dX1RerqoDr97WRJEmSpDYbZOA7Hnioa3uiKeulzkxtj62qnQDN+zH7KiU5Lcl24C7g0iYAHt+0n2kckiRJktQ6gwx8U10nVz3W6aXtcytUfamqXgq8HLg8yaL96SvJ2iRbk2zdvXv3bF8nSZIkSSNtkIFvAjiha3sp8EiPdWZq+2hzmua+0zV3Tf7iqvo68F3gZU1fS2cZx752G6pqrKrGlixZMuPBSZIkSdKoG2Tg2wIsT7IsyaHABcDmSXU2Axc1d+s8HXiiOU1zprabgYubzxcDNwI0dRc2n38M+Enggaa/byc5vbk750X72kiSJElSmw3sLp3N3TLXATcDC4CNVbU9yaXN/mvp3DHzXGAH8CRwyUxtm66vAG5I8g7gQeBNTfmrgPVJfgD8PfBrVfVYs+9XgY8AL6Rzd07v0ClJkiSp9Qb64PWquolOqOsuu7brcwGX9dq2KX8cOHOK8o8CH52mr610Tu+UJEmSpHljoA9elyRJkiQNj4FPkiRJklpqoKd0akC2vRv2jPevvz3jsHhF//qTJEmSNBJc4ZuL9oz3N/AtXmHgkyRJklrIFb65avEKeN3twx6FJEmSpBHmCp8kSZIktZSBT5IkSZJaysAnSZIkSS1l4JMkSZKkljLwSZIkSVJLGfgkSZIkqaUMfJIkSZLUUgY+SZIkSWopA58kSZIktZSBT5IkSZJaysAnSZIkSS21cNgDkFpjzzjcekb/+lu8AlZe2b/+JEmSNO8Y+KR+WLyiv/3tGe9vf5IkSZqXDHxSP/R7Ja6fK4WSJEmat7yGT5IkSZJaysAnSZIkSS1l4JMkSZKkljLwSZIkSVJLGfgkSZIkqaUMfJIkSZLUUgY+SZIkSWopA58kSZIktZSBT5IkSZJaysAnSZIkSS010MCXZHWSe5PsSLJ+iv1JclWz/84kp87WNsmRST6b5L7mfXFTflaSbUnuat5f29Xm9qav8eZ1zCCPW5IkSZJGwcACX5IFwNXAOcDJwIVJTp5U7RxgefNaC1zTQ9v1wG1VtRy4rdkGeAz4xar6aeBi4KOTvuutVbWiee3q35FKkiRJ0mga5ArfKmBHVd1fVU8Bm4A1k+qsAa6vjjuAI5IcN0vbNcB1zefrgPMBquqrVfVIU74dWJTksAEdmyRJkiSNvEEGvuOBh7q2J5qyXurM1PbYqtoJ0LxPdXrmLwFfrarvd5V9uDmd831JMtWAk6xNsjXJ1t27d898dJIkSZI04gYZ+KYKVdVjnV7aTv2lyUuBfw/8SlfxW5tTPX++eb1tqrZVtaGqxqpqbMmSJb18nSRJkiSNrEEGvgnghK7tpcAjPdaZqe2jzWmfNO/PXI+XZCnwKeCiqvof+8qr6uHm/dvAx+icMipJkiRJrTbIwLcFWJ5kWZJDgQuAzZPqbAYuau7WeTrwRHOa5kxtN9O5KQvN+40ASY4A/hy4vKr+at8XJFmY5Ojm8yHAecDdfT9aSZIkSRoxCwfVcVXtTbIOuBlYAGysqu1JLm32XwvcBJwL7ACeBC6ZqW3T9RXADUneATwIvKkpXwf8I+B9Sd7XlJ0NfBe4uQl7C4BbgQ8N6rglSZIkaVQMLPABVNVNdEJdd9m1XZ8LuKzXtk3548CZU5T/FvBb0wxlZe+jliRJkqR2GOiD1yVJkiRJw2PgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQZ60xZJB2DPONx6Rv/6W7wCVl7Zv/4kSZI08gx80ihavKK//e0Z729/kiRJmhMMfNIo6vdKXD9XCiVJkjRneA2fJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlBhr4kqxOcm+SHUnWT7E/Sa5q9t+Z5NTZ2iY5Mslnk9zXvC9uys9Ksi3JXc37a7varGzKdzTfl0EetyRJkiSNgoEFviQLgKuBc4CTgQuTnDyp2jnA8ua1Frimh7brgduqajlwW7MN8Bjwi1X108DFwEe7vueapv9937W6f0cqSZIkSaNp4f42aFbUTqiqO2epugrYUVX3N+02AWuAe7rqrAGur6oC7khyRJLjgJNmaLsGOKNpfx1wO/DeqvpqV7/bgUVJDgOOBA6vqi82fV0PnA98en+P/Xnb9m7YM96//vaMw+IV/etPkiRJUiv1FPiS3A68vqk/DuxO8rmq+vUZmh0PPNS1PQGc1kOd42dpe2xV7QSoqp1Jjpniu38J+GpVfT/J8U37yd/xHEnW0lkJ5MQTT5z+yPbXnvH+hrTFKwx82n97xuHWM/rX3+IVsPLK/vUnSZKkvut1he9HqupbSf434MNV9f4ks63wTXWdXPVYp5e2U39p8lLg3wNn78c4OoVVG4ANAGNjYz19X88Wr4DX3d7XLqWe9fv/IOjnirUkSZIGptfAt7A51fLNwL/qsc0EcELX9lLgkR7rHDpD20eTHNes7h0H7NpXKclS4FPARVX1P7q+Y+ks45Dard8rcf1cKZQkSdLA9HrTln8L3EznurotSV4C3DdLmy3A8iTLkhwKXABsnlRnM3BRc7fO04EnmtM1Z2q7mc5NWWjebwRIcgTw58DlVfVX+76g6e/bSU5v7s550b42kiRJktRmva7w7ayqU/ZtVNX9SX5npgZVtTfJOjpBcQGwsaq2J7m02X8tcBNwLrADeBK4ZKa2TddXADckeQfwIPCmpnwd8I+A9yV5X1N2dlXtAn4V+AjwQjo3azl4N2yRJEmSpCFJ5waZs1RKvlJVp85W1iZjY2O1devW/nS27/Q3r+FTW/i/aUmSpIMiybaqGnu+7Wdc4Uvyc8ArgCVJuu/IeTidlTdJkiRJ0oia7ZTOQ4Efbuq9uKv8W8AbBzUoSZIkSdKBmzHwVdXngM8l+UhVffMgjUmSJEmS1Ae93rTlsCQbgJO621TVawcxKEmSJEnSges18P1X4Frg94GnBzccSZIkSVK/9Br49lbVNQMdiSRJkiSpr3p98Pp/S/JrSY5LcuS+10BHJkmSJEk6IL2u8F3cvP9GV1kBL+nvcCRJkiRJ/dJT4KuqZYMeiCRJkiSpv3o6pTPJDyX5182dOkmyPMl5gx2aJEmSJOlA9HoN34eBp4BXNNsTwG8NZESSJEmSpL7oNfD9eFX9B+AHAFX1d0AGNipJkiRJ0gHrNfA9leSFdG7UQpIfB74/sFFJkiRJkg5Yr3fp/ADwGeCEJH8EvBJ4+4DGJEmSJEnqg17v0nlLkm3A6XRO5XxXVT020JFJkiRJkg5IT4EvyWbg48DmqvruYIckSZIkSeqHXk/p/I/APweuSPJl4I+BP6uq7w1sZJJG255xuPWM/vW3eAWsvLJ//UmSJKnnUzo/B3wuyQLgtcD/DmwEDh/g2CSNqsUr+tvfnvH+9idJkiSg9xU+mrt0/iKdlb5TgesGNShJI67fK3H9XCmUJEnSM3q9hu+PgdPo3KnzauD2qvr7QQ5MkiRJknRgel3h+zDwlqp6epCDkSRJkiT1z4wPXk/ymwBV9Rngn03a9/8McFySJEmSpAM0Y+ADLuj6fPmkfav7PBZJkiRJUh/NFvgyzeeptiVJkiRJI2S2wFfTfJ5qW5IkSZI0Qma7acvPJPkWndW8FzafabYXDXRkkiRJkqQDMmPgq6oFB2sgkiRJkqT+mu2UTkmSJEnSHDXQwJdkdZJ7k+xIsn6K/UlyVbP/ziSnztY2yZFJPpvkvuZ9cVN+VJK/TPKdJL836Xtub/oab17HDPK4JUmSJGkUDCzwJVkAXA2cA5wMXJjk5EnVzgGWN6+1wDU9tF0P3FZVy4Hbmm2A7wHvA/7lNEN6a1WtaF67+nCIkiRJkjTSBrnCtwrYUVX3V9VTwCZgzaQ6a4Drq+MO4Igkx83Sdg1wXfP5OuB8gKr6blV9gU7wkyRJkqR5b5CB73jgoa7tiaaslzoztT22qnYCNO+9np754eZ0zvcl8RmCkiRJklpvkIFvqlA1+dl909Xppe3+eGtV/TTw883rbVNVSrI2ydYkW3fv3n0AXydJkiRJwzfIwDcBnNC1vRR4pMc6M7V9tDntk+Z91uvxqurh5v3bwMfonDI6Vb0NVTVWVWNLliyZrVtJkiRJGmmDDHxbgOVJliU5FLgA2DypzmbgouZunacDTzSnac7UdjNwcfP5YuDGmQaRZGGSo5vPhwDnAXcf+OFJkiRJ0mib8cHrB6Kq9iZZB9wMLAA2VtX2JJc2+68FbgLOBXYATwKXzNS26foK4IYk7wAeBN607zuTPAAcDhya5HzgbOCbwM1N2FsA3Ap8aFDHLUmSJEmjYmCBD6CqbqIT6rrLru36XMBlvbZtyh8HzpymzUnTDGVlbyOWNDR7xuHWM/rb5+IVsPLK/vYpSZI0hww08ElSTxav6H+fe8b736ckSdIcY+CTNHyDWIXr92qhJEnSHDTIm7ZIkiRJkobIwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaqmFwx6AJA3MnnG49Yz+9bd4Bay8sn/9SZIkDZiBT1I7LV7R3/72jPe3P0mSpIPAwCepnfq9EtfPlUJJkqSDxGv4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlBhr4kqxOcm+SHUnWT7E/Sa5q9t+Z5NTZ2iY5Mslnk9zXvC9uyo9K8pdJvpPk9yZ9z8okdzV9XZUkgzxuSZIkSRoFAwt8SRYAVwPnACcDFyY5eVK1c4DlzWstcE0PbdcDt1XVcuC2Zhvge8D7gH85xXCuafrf912r+3CIkiRJkjTSBrnCtwrYUVX3V9VTwCZgzaQ6a4Drq+MO4Igkx83Sdg1wXfP5OuB8gKr6blV9gU7we0bT3+FV9cWqKuD6fW0kSZIkqc0GGfiOBx7q2p5oynqpM1PbY6tqJ0DzfkwP45iYZRwAJFmbZGuSrbt3756lW0mSJEkabQsH2PdU18lVj3V6advPcXQKqzYAGwDGxsae7/dJaqs943DrGf3rb/EKWHll//qTJEmaZJCBbwI4oWt7KfBIj3UOnaHto0mOq6qdzemau3oYx9JZxiFJM1u8or/97Rnvb3+SJElTGGTg2wIsT7IMeBi4AHjLpDqbgXVJNgGnAU80QW73DG03AxcDVzTvN840iKa/byc5HfgScBHwn/txgJLmkX6vxPVzpVCSJGkaAwt8VbU3yTrgZmABsLGqtie5tNl/LXATcC6wA3gSuGSmtk3XVwA3JHkH8CDwpn3fmeQB4HDg0CTnA2dX1T3ArwIfAV4IfLp5SZIkSVKrDXKFj6q6iU6o6y67tutzAZf12rYpfxw4c5o2J01TvhV4Wa/jliRJkqQ2GOiD1yVJkiRJw2PgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklpqoA9elyTNYM843HpG//pbvAJWXtm//iRJ0pxn4JOkYVi8or/97Rnvb3+SJKkVDHySNAz9Xonr50qhJElqDa/hkyRJkqSWMvBJkiRJUksZ+CRJkiSppbyGbzrfurd/18TsGe//DRokSZIkaRau8E3n6b/rX1+LVxj4JEmSJB10rvBNZ8EL4XW3D3sUktQ7n+snSZImMfBJUhv4XD9JkjQFA58ktYHP9ZMkSVPwGj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSPnhdkjS1PeP9fQD74hX9f0C8JEma0UBX+JKsTnJvkh1J1k+xP0muavbfmeTU2domOTLJZ5Pc17wv7tp3eVP/3iS/0FV+e1M23ryOGeRxS9Kct3hF59Uve8Y7L0mSdFANbIUvyQLgauAsYALYkmRzVd3TVe0cYHnzOg24Bjhtlrbrgduq6oomCK4H3pvkZOAC4KXAjwK3JvmJqnq6+a63VtXWQR2vJLVKv1fi+rlSKEmSejbIFb5VwI6qur+qngI2AWsm1VkDXF8ddwBHJDlulrZrgOuaz9cB53eVb6qq71fV3wA7mn4kSZIkaV4aZOA7Hnioa3uiKeulzkxtj62qnQDN+77TM2f7vg83p3O+L0n2/3AkSZIkaW4Z5E1bpgpV1WOdXtruz/e9taoeTvJi4BPA24Drn9NBshZYC3DiMYfN8nWSpP3S75vAgDeCkSRpFoNc4ZsATujaXgo80mOdmdo+2pz2SfO+a7bvq6qHm/dvAx9jmlM9q2pDVY1V1diSHzmkh0OUJPWk3zeBAW8EI0lSDwa5wrcFWJ5kGfAwnRuqvGVSnc3AuiSb6Ny05Ymq2plk9wxtNwMXA1c07zd2lX8sye/QuWnLcuDLSRYCR1TVY0kOAc4Dbh3IEUuSpjaIVThvBCNJ0qwGFviqam+SdcDNwAJgY1VtT3Jps/9a4CbgXDo3WHkSuGSmtk3XVwA3JHkH8CDwpqbN9iQ3APcAe4HLqurpJC8Cbm7C3gI6Ye9DgzpuSZIkSRoVqZrt0rj5aewnXlxb//rbwx6GJGk6+1b4Xnf7MEchSdJAJdlWVWPPt/1AH7wuSZIkSRqeQV7DJ0nSYPX7zp/e9VOS1DIGPknS3DSIu35KktQyBj5J0tzU75U47/opSWohr+GTJEmSpJYy8EmSJElSS3lKpyRJ+3gTGElSyxj4JEkCbwIjSWolA58kSeBNYCRJreQ1fJIkSZLUUq7wSZI0KF4TKEkaMgOfJEmD4DWBkqQRYOCTJGkQvCZQkjQCvIZPkiRJklrKFT5JkuYKrwmUJO0nA58kSXOB1wRKkp4HA58kSXPBIK4JdMVQklrPwCdJ0nzkiqEkzQsGPkmS5iPvIipJ84KBT5Ik9Ue/TxEFTxOVpANk4JMkSQeu36eIAuz6XOfVz9NFDZCS5hkDnyRJOnCDCFHb3t3fsOd1hpLmIQOfJEkaTd6ZVJIOmIFPkiTND/0+7dRTTiXNAQY+SZI0P/Q7SHnKqaQ5wMAnSZL0fHjKqaQ5wMAnSZI0CjzlVNIAGPgkSZJGwaifcjqIADkIhlLpWQx8kiRJbTTqAXIQ5koo7ScDrmYx0MCXZDXwn4AFwO9X1RWT9qfZfy7wJPD2qvrKTG2THAn8MXAS8ADw5qra0+y7HHgH8DTwzqq6uSlfCXwEeCFwE/CuqqpBHbckSVLrzIVQMRdCaT/NhYBrIB26gQW+JAuAq4GzgAlgS5LNVXVPV7VzgOXN6zTgGuC0WdquB26rqiuSrG+235vkZOAC4KXAjwK3JvmJqnq66XctcAedwLca+PSgjl2SJElDMN+CxagH3LkQSOeBQa7wrQJ2VNX9AEk2AWuA7sC3Bri+WW27I8kRSY6js3o3Xds1wBlN++uA24H3NuWbqur7wN8k2QGsSvIAcHhVfbHp63rgfAx8kiRJmstGPeCOeiCdC/rw32+Qge944KGu7Qk6q3iz1Tl+lrbHVtVOgKrameSYrr7umKKvHzSfJ5dLkiRJGpRRD6Rzwa1nAJ87oC4GGfgyRdnk6+amq9NL216/r+e+kqylc+onwPeT3D3Ld+rgOhp4bNiD0DOcj9HjnIwe52T0OCejxzkZLc7H6PnJA2k8yMA3AZzQtb0UeKTHOofO0PbRJMc1q3vHAbtm6Wui+TzTOACoqg3ABoAkW6tqbKYD1MHlnIwW52P0OCejxzkZPc7J6HFORovzMXqSbD2Q9i/o10CmsAVYnmRZkkPp3FBl86Q6m4GL0nE68ERzuuZMbTcDFzefLwZu7Cq/IMlhSZbRuRHMl5v+vp3k9OauoBd1tZEkSZKk1hrYCl9V7U2yDriZzqMVNlbV9iSXNvuvpXPHzHOBHXQey3DJTG2brq8AbkjyDuBB4E1Nm+1JbqBzY5e9wGXNHToBfpV/eCzDp/GGLZIkSZLmgYE+h6+qbqIT6rrLru36XMBlvbZtyh8Hzpymzb8D/t0U5VuBl+3P2GlO7dRIcU5Gi/MxepyT0eOcjB7nZPQ4J6PF+Rg9BzQn8fnjkqT5KEkBf1hVb2u2FwI7gS9V1XlJ3g58EHi4q9nFdB4JBHAi8ETzeqyqXjep/xcCnwFeW1VPJ1kO/C7wj4H/CXwLeH9VfX6a8b2IzpksL6mqJ7rK/xT4GJ0zY15eVe9/vv8NJEntN8hr+CRJGmXfBV7WBDOAs3h2uAP446pa0fX62r7PdK4d/41m+3U81y8Dn2zC3iLgz4ENVfXjVbUS+BfAS6YbXFV9F7iFzrNjAUjyI8CrgD9r+nt9kh/a/0OXJM0XBj5J0nz2aeCfNp8vBD7ex77fyj/cJOytwBer6pmbl1XV3VX1Eeis5iXZmGRLkq8mWdNU+zidG5ft8wbgM1X1ZHNZxO3AeX0csySpZQx8kqT5bBOdOzwvAk4BvjRp/z9PMt71euFzu3iu5g7TL6mqB5qilwJfmaHJvwL+oqpeDrwG+GBzSudngJVJjmrqXcCzQ+lW4Od7GZMkaX4y8EmS5q2quhM4ic7q3nNuFMZzT+n8ux67PprOdXpTSvKpJHcn+WRTdDawPsk4nVW7RcCJVfUUnVNH35jkaGAFndM899kF/GiPY5IkzUMDvUunJElzwGbgt4EzgKNmrtqzv6MT2vbZDrx630ZVvSHJWPO9AAF+qarunaKvjwP/uqlzY1X9oGvfoua7JEmakit8kqT5biPwf1XVXf3qsKr2AAuaU0Whc1fNVyZ5fVe17put3Az8iyQBSPKzXfv+ElhO5zFGk68x/Ang7n6NW5LUPgY+SdK8VlUTVfWfptk9+Rq+V+xH17fQuaMmzamg5wGXJrk/yRfprNr9VlP3/wYOAe5McnezvW98fw98gs7q4+RHOLyGzt06JUmaks/hkyRpAJpVul/f95y/AfR/LPCxqjpzEP1LktrBFT5Jkgagqr4K/GWSBQP6ihOB9wyob0lSS7jCJ0mSJEkt5QqfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqKQOfJEmSJLXUnAh8STYm2ZXk7mn2J8lVSXYkuTPJqV37Vie5t9m3/uCNWpIkSZKGa04EPuAjwOoZ9p8DLG9ea4FrAJIsAK5u9p8MXJjk5IGOVJIkSZJGxJwIfFX1eeBvZ6iyBri+Ou4AjkhyHLAK2FFV91fVU8Cmpq4kSZIktd7CYQ+gT44HHuranmjKpio/bbpOkqyls0LIi170opU/9VM/1f+RSpIkSVKPtm3b9lhVLXm+7dsS+DJFWc1QPqWq2gBsABgbG6utW7f2Z3SSJEmS9Dwk+eaBtG9L4JsATujaXgo8Ahw6TbkkSZIktd6cuIavB5uBi5q7dZ4OPFFVO4EtwPIky5IcClzQ1JUkSZKk1psTK3xJPg6cARydZAJ4P3AIQFVdC9wEnAvsAJ4ELmn27U2yDrgZWABsrKrtB/0AJEmSJGkI5kTgq6oLZ9lfwGXT7LuJTiCUJEmSpHmlLad0SpIkSZImMfBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS11JwJfElWJ7k3yY4k66fY/xtJxpvX3UmeTnJks++BJHc1+7Ye/NFLkiRJ0sG3cNgD6EWSBcDVwFnABLAlyeaqumdfnar6IPDBpv4vAv9HVf1tVzevqarHDuKwJUmSJGmo5soK3ypgR1XdX1VPAZuANTPUvxD4+EEZmSRJkiSNqLkS+I4HHuranmjKniPJDwGrgU90FRdwS5JtSdZO9yVJ1ibZmmTr7t27+zBsSZIkSRqeuRL4MkVZTVP3F4G/mnQ65yur6lTgHOCyJK+eqmFVbaiqsaoaW7JkyYGNWJIkSZKGbK4EvgnghK7tpcAj09S9gEmnc1bVI837LuBTdE4RlSRJkqRWmyuBbwuwPMmyJIfSCXWbJ1dK8iPAPwFu7Cp7UZIX7/sMnA3cfVBGLUmSJElDNCfu0llVe5OsA24GFgAbq2p7kkub/dc2Vd8A3FJV3+1qfizwqSTQOd6PVdVnDt7oJUmSJGk4UjXdpXDz29jYWG3d6iP7JEmSJA1Pkm1VNfZ828+VUzolSZIkSfvJwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRScybwJVmd5N4kO5Ksn2L/GUmeSDLevP5Nr20lSZIkqY0WDnsAvUiyALgaOAuYALYk2VxV90yq+v9V1XnPs60kSZIktcpcWeFbBeyoqvur6ilgE7DmILSVJEmSpDlrrgS+44GHurYnmrLJfi7J15J8OslL97MtSdYm2Zpk6+7du/sxbkmSJEkamrkS+DJFWU3a/grwY1X1M8B/Bv50P9p2Cqs2VNVYVY0tWbLk+Y5VkiRJkkbCXAl8E8AJXdtLgUe6K1TVt6rqO83nm4BDkhzdS1tJkiRJaqO5Evi2AMuTLEtyKHABsLm7QpL/JUmaz6voHNvjvbSVJEmSpDaaE3fprKq9SdYBNwMLgI1VtT3Jpc3+a4E3Ar+aZC/wd8AFVVXAlG2HciCSJEmSdBClk4k02djYWG3dunXYw5AkSZI0jyXZVlVjz7f9XDmlU5IkSZK0nwx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklpqzgS+JKuT3JtkR5L1U+x/a5I7m9d/T/IzXfseSHJXkvEkWw/uyCVJkiRpOBYOewC9SLIAuBo4C5gAtiTZXFX3dFX7G+CfVNWeJOcAG4DTuva/pqoeO2iDliRJkqQhmysrfKuAHVV1f1U9BWwC1nRXqKr/XlV7ms07gKUHeYySJEmSNFLmSuA7Hnioa3uiKZvOO4BPd20XcEuSbUnWDmB8kiRJkjRy5sQpnUCmKKspKyavoRP4XtVV/MqqeiTJMcBnk3yjqj4/Rdu1wFqAE0888cBHLUmSJElDNFdW+CaAE7q2lwKPTK6U5BTg94E1VfX4vvKqeqR53wV8is4pos9RVRuqaqyqxpYsWdLH4UuSJEnSwTdXAt8WYHmSZUkOBS4ANndXSHIi8EngbVX1113lL0ry4n2fgbOBuw/ayCVJkiRpSObEKZ1VtTfJOuBmYAGwsaq2J7m02X8t8G+Ao4D/NwnA3qoaA44FPtWULQQ+VlWfGcJhSJIkSdJBlaopL4Wb98bGxmrrVh/ZJ0mSJGl4kmxrFrKel7lySqckSZIkaT8Z+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklrKwCdJkiRJLWXgkyRJkqSWMvBJkiRJUksZ+CRJkiSppQx8kiRJktRSBj5JkiRJaikDnyRJkiS1lIFPkiRJklpqzgS+JKuT3JtkR5L1U+xPkqua/XcmObXXtpIkSZLURnMi8CVZAFwNnAOcDFyY5ORJ1c4BljevtcA1+9FWkiRJklpnTgQ+YBWwo6rur6qngE3Amkl11gDXV8cdwBFJjuuxrSRJkiS1zlwJfMcDD3VtTzRlvdTppa0kSZIktc7CYQ+gR5mirHqs00vbTgfJWjqngwJ8P8ndPY9QB8PRwGPDHoSe4XyMHudk9Dgno8c5GT3OyWhxPkbPTx5I47kS+CaAE7q2lwKP9Fjn0B7aAlBVG4ANAEm2VtXYgQ1b/eScjBbnY/Q4J6PHORk9zsnocU5Gi/MxepJsPZD2c+WUzi3A8iTLkhwKXABsnlRnM3BRc7fO04Enqmpnj20lSZIkqXXmxApfVe1Nsg64GVgAbKyq7UkubfZfC9wEnAvsAJ4ELpmp7RAOQ5IkSZIOqjkR+ACq6iY6oa677NquzwVc1mvbHmzY3zFq4JyT0eJ8jB7nZPQ4J6PHORk9zslocT5GzwHNSTo5SZIkSZLUNnPlGj5JkiRJ0n4y8E2SZHWSe5PsSLJ+2OOZr5I8kOSuJOP77kyU5Mgkn01yX/O+eNjjbLMkG5Ps6n48yUxzkOTy5ndzb5JfGM6o222aOflAkoeb38p4knO79jknA5TkhCR/meTrSbYneVdT7u9kSGaYE38nQ5JkUZIvJ/laMyf/tin3dzIEM8yHv5EhS7IgyVeT/Fmz3bffiKd0dkmyAPhr4Cw6j3nYAlxYVfcMdWDzUJIHgLGqeqyr7D8Af1tVVzRhfHFVvXdYY2y7JK8GvgNcX1Uva8qmnIMkJwMfB1YBPwrcCvxEVT09pOG30jRz8gHgO1X125PqOicDluQ44Liq+kqSFwPbgPOBt+PvZChmmJM34+9kKJIEeFFVfSfJIcAXgHcB/wx/JwfdDPOxGn8jQ5Xk14Ex4PCqOq+f/+Zyhe/ZVgE7qur+qnoK2ASsGfKY9A/WANc1n6+j80dcA1JVnwf+dlLxdHOwBthUVd+vqr+hc7fcVQdjnPPJNHMyHedkwKpqZ1V9pfn8beDrwPH4OxmaGeZkOs7JgFXHd5rNQ5pX4e9kKGaYj+k4HwdBkqXAPwV+v6u4b78RA9+zHQ881LU9wcx/KDQ4BdySZFuStU3Zsc2zFWnejxna6Oav6ebA385wrUtyZ3PK575TPpyTgyjJScDPAl/C38lImDQn4O9kaJpT1caBXcBnq8rfyRBNMx/gb2SYrgR+E/j7rrK+/UYMfM+WKco853U4XllVpwLnAJc1p7JpdPnbGZ5rgB8HVgA7gf/YlDsnB0mSHwY+Aby7qr41U9UpypyTAZhiTvydDFFVPV1VK4ClwKokL5uhunMyYNPMh7+RIUlyHrCrqrb12mSKshnnxMD3bBPACV3bS4FHhjSWea2qHmnedwGforNU/Whzfca+6zR2DW+E89Z0c+BvZ0iq6tHmj/ffAx/iH07rcE4OguYamE8Af1RVn2yK/Z0M0VRz4u9kNFTV/wRup3O9mL+TIeueD38jQ/VK4PXN/Ss2Aa9N8of08Tdi4Hu2LcDyJMuSHApcAGwe8pjmnSQvai62J8mLgLOBu+nMxcVNtYuBG4czwnltujnYDFyQ5LAky4DlwJeHML55Z98fg8Yb6PxWwDkZuObmB38AfL2qfqdrl7+TIZluTvydDE+SJUmOaD6/EHgd8A38nQzFdPPhb2R4quryqlpaVSfRyR5/UVX/K338jSwcyMjnqKram2QdcDOwANhYVduHPKz56FjgU52/2ywEPlZVn0myBbghyTuAB4E3DXGMrZfk48AZwNFJJoD3A1cwxRxU1fYkNwD3AHuBy7yDV/9NMydnJFlB53SOB4BfAefkIHkl8DbgruZ6GID/E38nwzTdnFzo72RojgOua+6E/gLghqr6syRfxN/JMEw3Hx/1NzJy+va3xMcySJIkSVJLeUqnJEmSJLWUgU+SJEmSWsrAJ0mSJEktZeCTJEmSpJYy8EmSJElSSxn4JEmSJKmlDHySJEmS1FIGPkmSJElqqf8fJo9dCVIxWfIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "from __future__ import print_function, division\n", "from collections import defaultdict, OrderedDict\n", @@ -271,15 +380,17 @@ "var = 'met'\n", "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", "cut = 'base'\n", + "sumcharge = ['ch+', 'ch-']\n", + "systematic = 'nominal'\n", "\n", "print('Opening path: ', path)\n", "hists = {}\n", "with gzip.open(path) as fin:\n", " hin = pickle.load(fin)\n", " print(' >> looking for histograms...')\n", - " for k in hin.keys():\n", - " if k in hists: hists[k]+=hin[k]\n", - " else: hists[k]=hin[k]\n", + " for k in hin[0].keys():\n", + " if k in hists: hists[k]+=hin[0][k]\n", + " else: hists[k]=hin[0][k]\n", "\n", "\n", "# Create figure\n", @@ -290,15 +401,24 @@ "h = hists[var]\n", "h = h.integrate('channel', channel)\n", "h = h.integrate('cut', cut)\n", + "h = h.integrate('sumcharge', sumcharge)\n", + "h = h.integrate('systematic', systematic)\n", "\n", "# Integrate over samples\n", "h = h.sum('sample')\n", - " \n", + "\n", "# Plot and save figure to outname\n", "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", "fig.savefig(outname)\n", "print('Output histogram saved in %s'%outname)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -317,9 +437,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} From 7496e41fd0d3df4799e193470a197fcc711c7841 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Fri, 25 Jun 2021 10:53:20 +0000 Subject: [PATCH 11/11] Move back analysis directory: we could use dask.upload_file to upload analysis files on dask worker --- .../.ipynb_checkpoints/__init__-checkpoint.py | 0 {topcoffea/analysis => analysis}/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin .../analysis => analysis}/btagMCeff/__init__.py | 0 .../analysis => analysis}/btagMCeff/btagMCeff.py | 0 {topcoffea/analysis => analysis}/btagMCeff/run.py | 0 {topcoffea/analysis => analysis}/topEFT/Genrun.py | 0 .../analysis => analysis}/topEFT/Gentopeft.py | 0 {topcoffea/analysis => analysis}/topEFT/__init__.py | 0 .../analysis => analysis}/topEFT/convert3lEFT.py | 0 .../analysis => analysis}/topEFT/drawSliders.py | 0 {topcoffea/analysis => analysis}/topEFT/genPlot.py | 0 .../analysis => analysis}/topEFT/make_jsons.py | 0 {topcoffea/analysis => analysis}/topEFT/plot.py | 0 {topcoffea/analysis => analysis}/topEFT/run.py | 0 {topcoffea/analysis => analysis}/topEFT/topeft.py | 0 .../analysis => analysis}/topEFT/topeftenv.py | 0 .../analysis => analysis}/topEFT/work_queue_run.py | 0 {topcoffea/analysis => analysis}/ttTest/__init__.py | 0 {topcoffea/analysis => analysis}/ttTest/plot.py | 0 {topcoffea/analysis => analysis}/ttTest/tt.py | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename {topcoffea/analysis => analysis}/.ipynb_checkpoints/__init__-checkpoint.py (100%) rename {topcoffea/analysis => analysis}/__init__.py (100%) rename {topcoffea/analysis => analysis}/__pycache__/__init__.cpython-38.pyc (100%) rename {topcoffea/analysis => analysis}/btagMCeff/__init__.py (100%) rename {topcoffea/analysis => analysis}/btagMCeff/btagMCeff.py (100%) rename {topcoffea/analysis => analysis}/btagMCeff/run.py (100%) rename {topcoffea/analysis => analysis}/topEFT/Genrun.py (100%) rename {topcoffea/analysis => analysis}/topEFT/Gentopeft.py (100%) rename {topcoffea/analysis => analysis}/topEFT/__init__.py (100%) rename {topcoffea/analysis => analysis}/topEFT/convert3lEFT.py (100%) rename {topcoffea/analysis => analysis}/topEFT/drawSliders.py (100%) rename {topcoffea/analysis => analysis}/topEFT/genPlot.py (100%) rename {topcoffea/analysis => analysis}/topEFT/make_jsons.py (100%) rename {topcoffea/analysis => analysis}/topEFT/plot.py (100%) rename {topcoffea/analysis => analysis}/topEFT/run.py (100%) rename {topcoffea/analysis => analysis}/topEFT/topeft.py (100%) rename {topcoffea/analysis => analysis}/topEFT/topeftenv.py (100%) rename {topcoffea/analysis => analysis}/topEFT/work_queue_run.py (100%) rename {topcoffea/analysis => analysis}/ttTest/__init__.py (100%) rename {topcoffea/analysis => analysis}/ttTest/plot.py (100%) rename {topcoffea/analysis => analysis}/ttTest/tt.py (100%) diff --git a/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py b/analysis/.ipynb_checkpoints/__init__-checkpoint.py similarity index 100% rename from topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py rename to analysis/.ipynb_checkpoints/__init__-checkpoint.py diff --git a/topcoffea/analysis/__init__.py b/analysis/__init__.py similarity index 100% rename from topcoffea/analysis/__init__.py rename to analysis/__init__.py diff --git a/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc b/analysis/__pycache__/__init__.cpython-38.pyc similarity index 100% rename from topcoffea/analysis/__pycache__/__init__.cpython-38.pyc rename to analysis/__pycache__/__init__.cpython-38.pyc diff --git a/topcoffea/analysis/btagMCeff/__init__.py b/analysis/btagMCeff/__init__.py similarity index 100% rename from topcoffea/analysis/btagMCeff/__init__.py rename to analysis/btagMCeff/__init__.py diff --git a/topcoffea/analysis/btagMCeff/btagMCeff.py b/analysis/btagMCeff/btagMCeff.py similarity index 100% rename from topcoffea/analysis/btagMCeff/btagMCeff.py rename to analysis/btagMCeff/btagMCeff.py diff --git a/topcoffea/analysis/btagMCeff/run.py b/analysis/btagMCeff/run.py similarity index 100% rename from topcoffea/analysis/btagMCeff/run.py rename to analysis/btagMCeff/run.py diff --git a/topcoffea/analysis/topEFT/Genrun.py b/analysis/topEFT/Genrun.py similarity index 100% rename from topcoffea/analysis/topEFT/Genrun.py rename to analysis/topEFT/Genrun.py diff --git a/topcoffea/analysis/topEFT/Gentopeft.py b/analysis/topEFT/Gentopeft.py similarity index 100% rename from topcoffea/analysis/topEFT/Gentopeft.py rename to analysis/topEFT/Gentopeft.py diff --git a/topcoffea/analysis/topEFT/__init__.py b/analysis/topEFT/__init__.py similarity index 100% rename from topcoffea/analysis/topEFT/__init__.py rename to analysis/topEFT/__init__.py diff --git a/topcoffea/analysis/topEFT/convert3lEFT.py b/analysis/topEFT/convert3lEFT.py similarity index 100% rename from topcoffea/analysis/topEFT/convert3lEFT.py rename to analysis/topEFT/convert3lEFT.py diff --git a/topcoffea/analysis/topEFT/drawSliders.py b/analysis/topEFT/drawSliders.py similarity index 100% rename from topcoffea/analysis/topEFT/drawSliders.py rename to analysis/topEFT/drawSliders.py diff --git a/topcoffea/analysis/topEFT/genPlot.py b/analysis/topEFT/genPlot.py similarity index 100% rename from topcoffea/analysis/topEFT/genPlot.py rename to analysis/topEFT/genPlot.py diff --git a/topcoffea/analysis/topEFT/make_jsons.py b/analysis/topEFT/make_jsons.py similarity index 100% rename from topcoffea/analysis/topEFT/make_jsons.py rename to analysis/topEFT/make_jsons.py diff --git a/topcoffea/analysis/topEFT/plot.py b/analysis/topEFT/plot.py similarity index 100% rename from topcoffea/analysis/topEFT/plot.py rename to analysis/topEFT/plot.py diff --git a/topcoffea/analysis/topEFT/run.py b/analysis/topEFT/run.py similarity index 100% rename from topcoffea/analysis/topEFT/run.py rename to analysis/topEFT/run.py diff --git a/topcoffea/analysis/topEFT/topeft.py b/analysis/topEFT/topeft.py similarity index 100% rename from topcoffea/analysis/topEFT/topeft.py rename to analysis/topEFT/topeft.py diff --git a/topcoffea/analysis/topEFT/topeftenv.py b/analysis/topEFT/topeftenv.py similarity index 100% rename from topcoffea/analysis/topEFT/topeftenv.py rename to analysis/topEFT/topeftenv.py diff --git a/topcoffea/analysis/topEFT/work_queue_run.py b/analysis/topEFT/work_queue_run.py similarity index 100% rename from topcoffea/analysis/topEFT/work_queue_run.py rename to analysis/topEFT/work_queue_run.py diff --git a/topcoffea/analysis/ttTest/__init__.py b/analysis/ttTest/__init__.py similarity index 100% rename from topcoffea/analysis/ttTest/__init__.py rename to analysis/ttTest/__init__.py diff --git a/topcoffea/analysis/ttTest/plot.py b/analysis/ttTest/plot.py similarity index 100% rename from topcoffea/analysis/ttTest/plot.py rename to analysis/ttTest/plot.py diff --git a/topcoffea/analysis/ttTest/tt.py b/analysis/ttTest/tt.py similarity index 100% rename from topcoffea/analysis/ttTest/tt.py rename to analysis/ttTest/tt.py