From 6478e1994838d7a1f7a75a9e4967b24bcbca66a9 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 6 Jul 2022 10:15:17 +0000 Subject: [PATCH 1/5] Add memory usage statistics for slabs and allocation classes --- cachelib/allocator/PromotionStrategy.h | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 cachelib/allocator/PromotionStrategy.h diff --git a/cachelib/allocator/PromotionStrategy.h b/cachelib/allocator/PromotionStrategy.h new file mode 100644 index 0000000000..1c4b6d9c8a --- /dev/null +++ b/cachelib/allocator/PromotionStrategy.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class PromotionStrategy : public BackgroundEvictorStrategy { + +public: + PromotionStrategy(uint64_t promotionAcWatermark, uint64_t maxPromotionBatch, uint64_t minPromotionBatch): + promotionAcWatermark(promotionAcWatermark), maxPromotionBatch(maxPromotionBatch), minPromotionBatch(minPromotionBatch) + { + + } + ~PromotionStrategy() {} + + std::vector calculateBatchSizes(const CacheBase& cache, + std::vector> acVec) { + std::vector batches{}; + for (auto [tid, pid, cid] : acVec) { + XDCHECK(tid > 0); + auto stats = cache.getAllocationClassStats(tid - 1, pid, cid); + if (stats.approxFreePercent < promotionAcWatermark) + batches.push_back(0); + else { + auto maxPossibleItemsToPromote = static_cast((promotionAcWatermark - stats.approxFreePercent) * + stats.memorySize / stats.allocSize); + batches.push_back(maxPossibleItemsToPromote); + } + } + + auto maxBatch = *std::max_element(batches.begin(), batches.end()); + if (maxBatch == 0) + return batches; + + std::transform(batches.begin(), batches.end(), batches.begin(), [&](auto numItems){ + auto cappedBatchSize = maxPromotionBatch * numItems / maxBatch; + if (cappedBatchSize < minPromotionBatch) + return 0UL; + else + return cappedBatchSize; + }); + + return batches; + } +private: + double promotionAcWatermark{4.0}; + uint64_t maxPromotionBatch{40}; + uint64_t minPromotionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook From c129a9c63d62f94d1fca490d883478e400f979d7 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Sat, 11 Jun 2022 04:17:55 -0400 Subject: [PATCH 2/5] Implement background promotion and eviction and add additional parameters to control allocation and eviction of items. Co-authored-by: Daniel Byrne --- MultiTierDataMovement.md | 115 ++++++ cachelib-background-evictor.png | Bin 0 -> 56182 bytes cachelib/allocator/BackgroundEvictor-inl.h | 110 ++++++ cachelib/allocator/BackgroundEvictor.h | 99 ++++++ .../allocator/BackgroundEvictorStrategy.h | 33 ++ cachelib/allocator/BackgroundPromoter-inl.h | 109 ++++++ cachelib/allocator/BackgroundPromoter.h | 98 +++++ cachelib/allocator/CMakeLists.txt | 1 + cachelib/allocator/Cache.h | 6 + cachelib/allocator/CacheAllocator-inl.h | 334 ++++++++++++++++-- cachelib/allocator/CacheAllocator.h | 224 +++++++++++- cachelib/allocator/CacheAllocatorConfig.h | 80 +++++ cachelib/allocator/CacheStats.h | 42 +++ cachelib/allocator/FreeThresholdStrategy.cpp | 67 ++++ cachelib/allocator/FreeThresholdStrategy.h | 43 +++ cachelib/allocator/MM2Q-inl.h | 33 +- cachelib/allocator/MM2Q.h | 7 + cachelib/allocator/MMLru-inl.h | 15 + cachelib/allocator/MMLru.h | 5 + cachelib/allocator/MMTinyLFU-inl.h | 7 + cachelib/allocator/MMTinyLFU.h | 3 + cachelib/allocator/MemoryTierCacheConfig.h | 4 + cachelib/allocator/PromotionStrategy.h | 10 +- cachelib/allocator/memory/MemoryAllocator.h | 3 +- .../allocator/memory/MemoryAllocatorStats.h | 4 + cachelib/allocator/memory/MemoryPool.h | 3 +- cachelib/allocator/nvmcache/CacheApiWrapper.h | 2 +- .../tests/AllocatorMemoryTiersTest.cpp | 2 + .../tests/AllocatorMemoryTiersTest.h | 92 +++++ cachelib/allocator/tests/CacheBaseTest.cpp | 2 + cachelib/cachebench/cache/Cache-inl.h | 51 +++ cachelib/cachebench/cache/CacheStats.h | 97 ++++- .../config-4GB-DRAM-4GB-PMEM.json | 8 +- cachelib/cachebench/util/CacheConfig.cpp | 49 ++- cachelib/cachebench/util/CacheConfig.h | 38 ++ 35 files changed, 1745 insertions(+), 51 deletions(-) create mode 100644 MultiTierDataMovement.md create mode 100644 cachelib-background-evictor.png create mode 100644 cachelib/allocator/BackgroundEvictor-inl.h create mode 100644 cachelib/allocator/BackgroundEvictor.h create mode 100644 cachelib/allocator/BackgroundEvictorStrategy.h create mode 100644 cachelib/allocator/BackgroundPromoter-inl.h create mode 100644 cachelib/allocator/BackgroundPromoter.h create mode 100644 cachelib/allocator/FreeThresholdStrategy.cpp create mode 100644 cachelib/allocator/FreeThresholdStrategy.h diff --git a/MultiTierDataMovement.md b/MultiTierDataMovement.md new file mode 100644 index 0000000000..cdfe6fd76e --- /dev/null +++ b/MultiTierDataMovement.md @@ -0,0 +1,115 @@ +# Background Data Movement + +In order to reduce the number of online evictions and support asynchronous +promotion - we have added two periodic workers to handle eviction and promotion. + +The diagram below shows a simplified version of how the background evictor +thread (green) is integrated to the CacheLib architecture. + +

+ BackgroundEvictor +

+ +## Synchronous Eviction and Promotion + +- `disableEvictionToMemory`: Disables eviction to memory (item is always evicted to NVMe or removed +on eviction) + +## Background Evictors + +The background evictors scan each class to see if there are objects to move the next (lower) +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundEvictorIntervalMilSec`: The interval that this thread runs for - by default +the background evictor threads will wake up every 10 ms to scan the AllocationClasses. Also, +the background evictor thead will be woken up everytime there is a failed allocation (from +a request handling thread) and the current percentage of free memory for the +AllocationClass is lower than `lowEvictionAcWatermark`. This may render the interval parameter +not as important when there are many allocations occuring from request handling threads. + +- `evictorThreads`: The number of background evictors to run - each thread is a assigned +a set of AllocationClasses to scan and evict objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is 1 to number of AllocationClasses. +The default is 1. + +- `maxEvictionBatch`: The number of objects to remove in a given eviction call. The +default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and it might increase contention with user threads. + +- `minEvictionBatch`: Minimum number of items to evict at any time (if there are any +candidates) + +- `maxEvictionPromotionHotness`: Maximum candidates to consider for eviction. This is similar to `maxEvictionBatch` +but it specifies how many candidates will be taken into consideration, not the actual number of items to evict. +This option can be used to configure duration of critical section on LRU lock. + + +### FreeThresholdStrategy (default) + +- `lowEvictionAcWatermark`: Triggers background eviction thread to run +when this percentage of the AllocationClass is free. +The default is `2.0`, to avoid wasting capacity we don't set this above `10.0`. + +- `highEvictionAcWatermark`: Stop the evictions from an AllocationClass when this +percentage of the AllocationClass is free. The default is `5.0`, to avoid wasting capacity we +don't set this above `10`. + + +## Background Promoters + +The background promotes scan each class to see if there are objects to move to a lower +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundPromoterIntervalMilSec`: The interval that this thread runs for - by default +the background promoter threads will wake up every 10 ms to scan the AllocationClasses for +objects to promote. + +- `promoterThreads`: The number of background promoters to run - each thread is a assigned +a set of AllocationClasses to scan and promote objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is `1` to number of AllocationClasses. The default is `1`. + +- `maxProtmotionBatch`: The number of objects to promote in a given promotion call. The +default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and it might increase contention with user threads. + +- `minPromotionBatch`: Minimum number of items to promote at any time (if there are any +candidates) + +- `numDuplicateElements`: This allows us to promote items that have existing handles (read-only) since +we won't need to modify the data when a user is done with the data. Therefore, for a short time +the data could reside in both tiers until it is evicted from its current tier. The default is to +not allow this (0). Setting the value to 100 will enable duplicate elements in tiers. + +### Background Promotion Strategy (only one currently) + +- `promotionAcWatermark`: Promote items if there is at least this +percent of free AllocationClasses. Promotion thread will attempt to move `maxPromotionBatch` number of objects +to that tier. The objects are chosen from the head of the LRU. The default is `4.0`. +This value should correlate with `lowEvictionAcWatermark`, `highEvictionAcWatermark`, `minAcAllocationWatermark`, `maxAcAllocationWatermark`. +- `maxPromotionBatch`: The number of objects to promote in batch during BG promotion. Analogous to +`maxEvictionBatch`. It's value should be lower to decrease contention on hot items. + +## Allocation policies + +- `maxAcAllocationWatermark`: Item is always allocated in topmost tier if at least this +percentage of the AllocationClass is free. +- `minAcAllocationWatermark`: Item is always allocated in bottom tier if only this percent +of the AllocationClass is free. If percentage of free AllocationClasses is between `maxAcAllocationWatermark` +and `minAcAllocationWatermark`: then extra checks (described below) are performed to decide where to put the element. + +By default, allocation will always be performed from the upper tier. + +### Extra policies (used only when percentage of free AllocationClasses is between `maxAcAllocationWatermark` +and `minAcAllocationWatermark`) +- `sizeThresholdPolicy`: If item is smaller than this value, always allocate it in upper tier. +- `defaultTierChancePercentage`: Change (0-100%) of allocating item in top tier + +## MMContainer options + +- `lruInsertionPointSpec`: Can be set per tier when LRU2Q is used. Determines where new items are +inserted. 0 = insert to hot queue, 1 = insert to warm queue, 2 = insert to cold queue +- `markUsefulChance`: Per-tier, determines chance of moving item to the head of LRU on access diff --git a/cachelib-background-evictor.png b/cachelib-background-evictor.png new file mode 100644 index 0000000000000000000000000000000000000000..571db128b2b7fd45e12a83721b91674abe42967a GIT binary patch literal 56182 zcmeFZWmFtpw=N2elK=_s5F`Y5*N~u%1r3b{cXtm71P|^W+}$M*B)Gc<_uvkvnpeK; zd-tz<&yRD)*kojMS9R5@wdR`g%xBK-Km|DoG*m)V7#J8dDap5rFfed!FfbruWDszs zMp%IXc!9N7ln{j}86@5Y4n7&amok!-g`or9Bf}uTV#2^dF9H6+z!JhBJidp4k%lGu z_q`(QtG}-S#tblpf&cp&2=EI1lLY<)cmMqgmk#@%5z|5cTn*Qj4)@P{5Ha*N73qdc z;047-Qr#W~<_Q+`KP*gQGCmB92#nO**UB!idnrgB$||z~)^4K6Qj#y=G2ITE>15$M zE50;0?iQGLYpJPCmFzY&GVMU7RHm$_D(G-6z92kPa66>V2%r+d6m`3~k1CAQGh#Ck z#k7hv_PXt|A99sLWV(8&e_jXAk@n{y@}&q2>^>ZW%!|LrhTiPsv%&tyAq*@T zFt(@9XWD;f09^HK*aPvO2|;gKhJ%H#!YE_;{GWOHzD0LDIt#m)J5j=*$9 zL{13)nKE?tJ{Z8*LT&BX|5*tPun-}*KX3Z&3j>-(dLJ+P$9iF42aNwb1PK9S-@rUk zdKB8ygaQysXK? zq5FvJWXK}vUVy&r#&I{lhA!Zv&+Wka`noo82kPcmy!yJjX??f?ZPlye_p_xs!ugwi zNEp&m?i_hmMy2XqzTN}@Mk-T_%s;-Qy7bpw`q(;Lm4g)@mH#CyyhBHmbqFEonaNxD z2B-ak1sYLNQI1?-216(XTu(S~cq7A!d4CDebXpW?*4r($c-Gg~_mc728Qh%j>woyb0L$22 z*)-T7BO^o1?^J|cGuuBn=w$VcYs{B1L`aA5K0|y~IP?Wk=xwOKPI|;CzkZ*i_>@{m zhYlf5XV{OfV6<)*Z!hZmt_bhtfwa;qW*`dtBFgw!?h*bT(BFFIBSJ~8+xy}3%Rc-0 z0+qt#WMbZ=l+EnEfh2Bv8eU%Y@1z+S8PfdL)^Ul6WLcpArLmgAI=zQf|B2b+FJ_o9|(0V9=i?-1=>8`sZp;BR=bgPfdd=vT9(Y zVhVP~w5k{-8Wiu(=5a*6j3lA8_}eaV=Jn*AJuXuwj-THp>Kn9JEAxD|?bfk^?tJE-$_kJZ?qX&P1W`-RYQY5|`8^-4}H$qfHbkHLxIMR2)EYiW-)&m?86xGKETwnw8*%PK>$jn- zZ1?V(5>n2*TnDdY1eu~9T`gKHu)++SaX=|r)O-f2hjU*={lrs&u zgcKDOF>2IoHM^c{Fr6F>XS~^+ERM;?HyN9Lx@N;#nB9k(MK$((iI{g5S&jp}N$>Q& zpQpKJ>nfG9gwnaZ14FBBtVi9YsPRjNxQn&^;gGv6F+DjH-KA$>l7Gs|Smf49hv-%W4q_LZuo)lt^M4dR`c`!OQrWO05 zpt{BL<|P%?`%z8VpV`e;3k_9O9Gfk2$=Zu^dBZ=y;B>Wbgni=rHSJdU9A7V**uj7h zHmvir;zZc#5c7WLk>?kEgcpwn5c(MqMftqc1}rca8PI5B(6GXqzklg7^nLDJF<+I&9+8_JeJ4u$vQgJ14O1Wq@lVI=|CD1=Rcv4Y_AgTyi(6 zPVC_fTEKU zWW*L;L{+**yh%KHRH!=W$QO5pPQb_=c4-BsVQsk+FeoJFN z8t7RWvPmBoy=*(buWET$ZC`WJxC(7;eo48aR^trR|nvMrp||nV|!k4EQV{KWo>k-y zAAKG9uoc_~0jQkKQrbq>z_GD)HE!a2XJ@0 z4+#8$fy|%^t;#Zr{KsY~z3}m|qH|7k1;7AWaNFh4DvfL+HvSglFcU59c2VWv&SbF$ zXKv1<=$ki%rXd86wgoxGB7Z@^0(lr%El<_=P%vWh75?MLkNy~B zB^h{lcsEy9+ZEtACTeQxLbXz>uP1uLP@&0j0zfXm1@07xOVeLP{t4<=Tvf|8;%g=y z%)>p2`q2iLs#7d$)=bRI7r(4kVf=|l#l*x&HB=p4qXqe5V(Psgyj)Y9_h++lv57J9 z@Z#d*zxbR>M#?}V9}Sg{kI)N>r$E@rkw{l}rex`=HIP9`Mae4vRJGY?F4Foi!LAfQhK)qSqUvQV|SVm3THJdMpTsGa>= zD+Z}Roykb%$B&>wWt8=yB<`X9i1`iY@zD9G zw@k=t?Q1Wu7CfS$N9$^P0su2S9&7+}$;ON5KFJ!2^Y_k+IG5M&5vTOdqU567cJ7MZ z4Jj$b4`}6}3nB>&#UrZsUm-Jk`;;SSjD+{IJMqD0MY351`}$$Vn*bS_a^jy7VZ^p2 zg>8n6mvgn&I#wlaY~o?WGcBI(fx2LkzV8gBl}tD>tWJCHX)US~Qd0Kkzr@69ThG-L zGdUkFS{g?Fm2IRUmxZRt*RlA+z$ZTS9UQ{{Z-V@W4NDve>n~>Z54i-{^<5dv#BV_m zo<+aakjM`4IexT>;`6@fP5^Em6W0Qej)q2J&}ehEVZO<=68C|chK7E+<7u-EdYghx zC2h`n2bYHVNT!{@V6k#ewTHsBhLy!n^G@Hom?$7t02(ww;e*rwiDv;EZz2V(Hv;uz z@VB>S9X;||md*NqM1N(bZF**h=$v_T18pC}HpF-Qx{u|f73)5W z#-1D>SCxxPO9%C*{1-r?#bglok{^Jn4UXo9bcnMYTyVU}1O8}4Ir>kihw&KRe@#f( z)_e;9d_ZwLs6~(iN`I6A9ySricfC0jHRVA~kHO+V(u6Xsn48G=x%kiTy|P>&;@~IQ z1u5L~U+p2bql!b>0iK|5?6MBHwG`Z_PoAt>sA|RCvC3GNtD>dab`kJ>nGRYUiE{3R@{e%!{Ih0|b!bPb3- zIsAreY;;UD-=1+4`h0Q)JSf({Q#Qr=18a-`GvuDt==<@gI?%)r27c!w5Wbl5MqfTk z4X`o*Yh+OaSR<>sTIMO}na#^Gs|CJIaJEJ#;S!Vc&%#{r#rJWgZil~Yq55^(kgI#v ztjSj{=%CYKZ4zn@#ACF*P8}aav{Cw(A_UJEyMMAZJ$@Pr^Mcr^kun77Ka*;kgl1_NXM`qW@XW ztoH8R2z>59G!!-fL|r5L+R{m{e=R9K@$%u+@Minv;|B~-I9PPkX#zdsL0%(fyjnFU z!XTlQn@K}@{>c6083Qbz8={;u)Yjx;MWjrxIvy51mTe*kA!%-F21!{$kA`Y^U;t4 zVSC0dNN7sW~gfRz!Y&TB7C*0U92x^UT&KR&01h!58d=BG&4%CmtoRq3e!)Bo>O7(tC=Cmo`o50r1r3SZAYIy2ZbXf2@qGt^i6 zD#?o+5j6)c@tj@wJ_lwm8wU)&z36TP&6X8FvkhUFp@?pW_*}r+q4f!(ZchVT=sr=T z7w^7*J^kr`TL~cZ_5jY(vemu)3v~XK*hBlrjNV4 zk7@J6`WcK?ql_t#0{HlZ13pIx)Bz9?K5ewrZRo?f*up7LC~os?nMza$2`XbkOjy@+ z8uT!0NlK%l&|r}V(;Ib>se=y5FqpbQ!bDoFLN_`52Yl1L>y6~V`E!x4jO^{d(ozSZl_ z{jjIX!FtsX*L1T6na1d)&(dy{jQMuoq|Gbud8)5 zBCe+U>s{kC&+|KgT&>to&ASKkI_vuNSg~N-V(sQ@vWH6v-~n7#^Cqf_8+q@tQaz4; z0zBzpGVe+I6K0Rwqi*^44w>k*05&2Yb^wTjq=RPT4Paqf&+34oy)Rb+lB^nNX=rGG zdC9Z3^_ZH*Sx;{zn7PG-aNg@?AGp#^m|4BT07!~R z_?{00e8e z(Dhab0lVg`th~JUPJS-7wCzFt=Az@27QgdB)l;FHJ$71MF;AH^AXT>sie z6~NF`B3_U|jcy2jV<0J)FGyHOZvfcByNm$R-rioty<_$v0d|Ki1=k`Ka71t9xcOvJ z#Hc5*+1kLf3fP(*%H1!lV(8}84~@vXzdWa8xPgU3dhKf`g;7Goub04}}5`PlkjaIxvKvqM|AmiAhR+X=kaT zP(UPQFWkw@%oJTTiP3f#5p#Kw(Zh zm)!e@Uiwmw&wCM+5&*6L@&*nGeXAKD8iUmCJIL;BE`P5K4=apy=^LqNnINqE>I}|K zU0;h0CGaA7_DpvsX}h9=?a!t7_&6DhqC@wP5YuF;B*hHk+ouf~i4+p!F+1SJK9^iLk6 z+CehlMQDH#X`=)(Q3(cv$MOY=FOY2)&L;EPH>t%Y3$BK+S1lzQ96%B4M2aC95mBSf zis*iBM)PlFw+=M!ydM1=l}_oNqawzgwW>)`aV6ryo>RTU>GBS?xdz9=(ZIk!?H~sI z$2dj4EC(p$;3s0JnQ>Ct%oxVeI;r1u7i-iGnu+qi8A{~rYfu)Q`(T2{8@LM0-EBDi z^~lJ`V79cma42)@tz!8&UuY-YalB~^VA`rga)lSAt@jOp^{ckr&Y%`dzW5R}!@};^ z6YR7T5AXwV6oq{MoksYst*rsx;37hTbfg3?PRFaBcr=OIda{eqIMp?0kg|%9^P+qB zB@y4jTOw|H(GEI7E{ph(>=;L*40N_+7QK%_0!9#(=wZ8#pHVdC@)&2w@7DVmCPwJ& z#N<=>RP^gEe+4twErS?UZf{S9n43=(UDFL$0mJnCZw!G-o-BHOnyS<;2D%j~Z>|b$_*~GNPYz zB7!qnq{SIcsa9KAt8x7vbREjQI4oIpROxgAl!%6zMoOEGgVYzm5%xEK6Cw#dY4sMJ zyupBX7+yv`O*@y-ym^6UHI4S6tyw~lYxUy{Cw>8!%cLA33JOY4I!mFsI~t0*n0kDi zVzZsF`GTIVZu)5Cte2M;ouZhSr%6PbwrVRWd-3QF)p%o=>O1G4C7bdK#kCLHM|Rf% zQRe8!{Ugh6N_kU>x zbotfcs*Mel^Jq(HJH?=BG>-f$*^-4u0Y)Ii_1B>`wAF zURSeam3{Tu`Z8-;pxX6@k_mqLbH^jmCh~^j84 z#O=eCZ-*ESZ+VH-fVaEtW?10L;lRSpj&Y-01!n;Oz0ktiik0|NS$PIm%O1`TY8FW- zb5l|;&yGV{n<@vgWI5Ss&_Y{HERvjxiCb@<1oyS5M_i3|=XYkWPZWiz-vNyqQy9i> z00!ClBqH`%&$INc1bkum@o`Xc6DGAZn7+4{UYT&qyTaS-3F_~giD)M!dGKkegD?_& zvW1^x`$utr;^BL#P#_^4_@-xv`5j|B$-zp|{D;b+W7Mp?IvG>JkNwGs%>e<3w+E3- z1@2Yp%0o#;8NQn#k_y+=Lsui(?3s=#Phhpw3{3HWD1-zG2*cL7{moXHbyKp(XC*Qp zoc7JDf@NUGZRZZW=3Dc&CV`BAG(M@cRJ3pBJw>LvuBp1%>xk9hVYf3AJXT8?B11&P zO9X_>R7Msm^5H7vrdz3kd(%P7)%L|Y=jDi7%-yT3ezHljd)>wR`HzamuU@>qce7f< zA-#}fMLX>i5MWRczFbEx({_8$IG0S#ecm*Iq;io@9Z&Ye2vwdcdL-V<77Br&+OH0Q$|9m%ZE)Vy#f)S7pTH0xA(TKSOo#fdwsKepOXPvJ=~V z^7B?Ma+l_ZaJr*R9(8(}n{m>ItX(H!XzIWL3)+B_@SbWgQmn`^x5^)BFwEmT12pRT zgKYy7POhFIAI+(8xv=JlN+Hf!$9*LkNoA_>iInkTQg8ehP2^9QIZ8>&(c4xZ)w z+FG;n3eKNd2ophQV+W2GS~HfqYTc0`0gKaB{8?>gzMnjil?hz+(_XM)J|cX1kBO@*Oc2oYh|851BfPaJVtUvvp)JdPb$}SuCS9SLS=A5ci)os=z3KP zYqsHfcuT#jR`2t0E`wA%kW0G!6{9W;AY;#G(`D?=V&fok3_JyxBJw|BItvxcJN7W3D8pLk5@a6107_2HfhYT**A3S%2=& zF94Ly`9jwprGXgE2$4ipxnwkAu~{FI`$8}?;LO3|rHUfte$~HYqs4L1=)lvQQh?To zURZ!bVd--aV!53pfXZZpe6Es5N`wY0xk`KK;Zou(2k++|YVLOSoURMw>OU%Ij1NVvK4NL;ND5oGk*#oZ0!8Y?DF;tT2cLhPkeES)L+`EE}L~ zf&0V+3p+6bYB%F0BzUje=bX!iv)P^5@!*&qsT!=`_}Ahe55>!R~%uAqCA9u(IWq%pEF?YTJq`Ji2s2^>9#;f<6y$+K@C-iD>fgr#9uf(7#%f)7KL}L)hFnn4 zvnzx^qt>c8Wy1skn25*(ywLsSs`taql1iKs2=)dOx$1BjAE=)y#K8lJ_vKHTv?54# zL1yNM-^<^pwB4bFLN1GGn<*{l&jA5IWwQE!@8J-*|7tUB2}qwQ#Y4*6XzA(QPX@U0 zm^59shBIFKA>Qq%wVI9PgsKB+bb%aDQ7vQi9UUD75^(43(d_*Ee1LioJZw4{wWXZqDJ`#F-67EAAwl8vPniUZnPB^ zfR$lDTWC4NKtM)w+~x}hB%&>ctq)%Jw^tX7?$$u23e+^;Pcb+h%=5Y)uN9UwNH668 z=}ApP)Z3QZ<38%d*6SU5&lSPb;Wt2;ak0`k{Am-m`h*l9?HcMbph1dNziaEkP0;BS z>+WnyNBCy15*Yu!fwW8$4#)$ffka-%i<{&Z6dKRH8}Pb$SX}}DzaPX(204zMnshBe*XwGOs#EhPSHN8UdokEt2Uq7 z+3n5FVs5zNM|r*92EuJJU~K0Nwh-m$cqrlX+dP)9+Oooe@y2Ffk4CdXMGE(yNs;twn;b&-V*&^YUH^t^>5hM0y;> zCa&Zg5nxpwPoW_175+Qht`vUfR1VX~i-g@^32)Q8^Eqp34?JdVj|Io6ZRS)X9M|9> z!HapINnoo{!l<l2h|l>sF)_nF>GS6+$)-6tUl#x>!fAgtA@23-*FewRY_)kD zQ!{pb?Hhnu(4_~RYz&OKST&zbVukITj>)U-sj8~RR_zJjAB6+8Ip#|QEvDe zwv#Pc!mvpPO<5bCEBJi!S583+`^9%!xOj@YTiZa&iMh0y=Ix~=V97zXS%+S!)cI!D zR7oXbp^Yt72_PFhd#z#Ne0;3oh1BC~Jiw}wU%e%5d=~xuZlhL7yzz@Sz_4E`io#Bn zm8-PbRLjymMC~9+OnU1k-cJqhS1`OUSIHjo-tP<#4;Qe-=%mDfDBwV#WX6HM<6}F( zLe8NRBf`wPhm!!4_a6Z*dq1$}AsHFT4KA~J>YGSm36pMy>}+%T5oj}P-;}^U&%%`Y z(iUGVv->WRXfQ0sb9xVC@b<9@-L1443lnc8jyGKw$LnFLj#tD5UGqg?N&1f1v3GZM z^)6X-iU14`T4N^u*+S0Qmkk|=ppl#DQoSVIyv2`RhF7nCUtGiuUnm+7L;((YPb&Qy zgSeQKRM^X-(rDn+`wjt5GkrTAgZNP)C#cdTf*4ZBc-&@RNeG!Ti%nP%SLVC{9J<2+ zC%YvTd}09Nk1FJvmnV&2K4+^{KOZ?6nQ=u@M&T)^K2=ImtXTmSZ%Gnvt9igN<`%P8 zsZ|-i6f71oTQ_&6vy^rGml$#v^%j){gV|fVWWXdFkzc=(qwQz+xF5>8~nqk&lNZ;&<9B>+jFXI<+``T%eB> z(5g=Z3qtd6t7(>aI9+0FO(@Ti@5bFoXC_%P+Vfe(jZd>-1Govk8ggdl!&LW!cRrsC zAHE}z)$PF*ulrpqZM5E>F*E)EXf)n$jQZ>^EQ`ra@5)s+Dc_4hNsI!_0u^QDZ(}Ml zaGriUyRy63j@#LJ1(Q7hgKHtWT+zogmR44Ubplvb(F@bsUiC^N^*Cu0ji>t*xoN+k zMSQ2B;T;2@j4vqkUcjh9*`ak@XQ=Aan{G+Cz6wVJo|*nZz!U2lhjH(j}75 zHuqCdAD3~nu=NszhXkMNU@dOPToeAX4@eBIWA| zH60b6>U-6~Q|={?tS@iug`atvn;jZ^dhR5tcy~;AIRzC?8ac6@D6Wn201Y!hr`Q&2 zX0tP?>)P{uw~UU+(}zYmg;7#MH=iv5QS&N3bOlq(?J=Mch9uj-s@gQx?Q!QI>{xE; z+Zcw@N&N0jzuTWkgNNzm^moSd3H_Wlm5aW_((UBl07Nt$Mimgf5(bXZVWWU>BM*Y# z1E*IN-8RM`QK!J&-sOZ!5et(uk+%&IAB1i?ZpDtzFFoXf(>BOSib`=U8P0$RM_-}t zcvhU_ndQ(Mh+WAvz{xRNsA43q*0P4hx0!5DRWT4&)iIv-aBFNoXXF``Km+tDJCMdZ zO(^7I4jXOE#V~WyKQBd77rx%gO0sUb8stBW4sQa`2cNLmoKMdvK%1>XP2C;S1z_sW zK8;9GUyqcux^35%pLdVLz0I1)>hhTF5nv^ z5}C(vFGua@SlBq`FzWLW$u*v@^skA`cf#%XDu=58&E^yXfhd-Zi%~&HfKw&|xgunz zG0Q#tzPbBZ+|EQgWn@!pa;Tqlzp_Sv*MDxcT0VYyDK0@~t}{Y%4T=il++Rl<6&gM| zI=Z&c#V)%iG~eixajf1GXD&1Zf{>{r=B&81w-#sIaow~#tzEf8D|XshFsaQD&87je z!Cy=g%^1j$h{Bgp*z(ka$K$eWl56Y1*8bOQzl7+dTY(AACsH1WbDnV+!2E6a^{o1J z>gUJ{Oxyr%H4`cM<0-ptw02yDbOUg?bZj(H*0o6Jq+3z-N#mbI_OW_;=&oH>?q|Sk z{-xs$TSF>K9cz3^-3hcRi`H9D2cdlgpbF~IalL>Er>0D_&)a9jlH0PQLoI;!Pp^(={ z0z`v5Oj;NL(2L^~zA+k=OG%B^YHs(;;3cnJZBtkjXBoy6nY)IQ2w zEpGGuo~vAMO5Rkss=J)iLpJ&87_>|T0P^WHplR*?;>&X`IuMe5kPzhS`Fi9jz&DUK zmq={Z2oY0E=ZUZo{55S6KcEO`RaP73%gq7GH+7vIaXs=c3iMmmZYlKk;4p_Lnq@Qq z7zOCizS7X|Lm@xQvj-K{@J=G<JRI&M^O8J@0$_o{V06|TyJKUM z_(2^Y_tm9~Od@lg$s?RRPT5yn1c)KXRuoEQB)%TS&ovW| zR&G5}STm+%)`Ea(EJN!K;cYAdK58Hc<>k+Ij+A!6GV}2_Q|DWW zKL}r!Hof{=XdEJd&=6h5@GBTm_5{9`+QWOl_jb8Wufxl|0Gkk)_35#_dXxgNpalN7 z30E=)8)=}*CQs|dXN?U%X)!cH_ zueUcqAgLI?#5nv(_}cnn-Q@K)K>9rNTHjQ9r*X?f1LA4HT0Z*4&u%u34&+4pHs4}i ziZ1_-_!UlK4v%rVi+$BLvCu#*!4VOd*3SXZXx`$g5sRg%(H?C33INyW!3CpI{-nmbT*Qy{Nx6wH4t`PO|U>+qQe<#X@dJ4Q?P7*diwVy7xA1Fx+$RK zP+sp&M|B-1R>)hT6{&PP%&i1n@hadHjRxFIua!bK4b_Z1bl?d<@0l#g^E?_@5*%Fj zBOm?}lJjnuhxDlAK&r&V28U&byT-DI;w-*YEbQfzh#sv~Bu%cZl9FP?)HPkR=7wh} zQ=3yXV|zG$hxZ*nysMEdUD-m%>#Vw`wbjs?Yju*w%;rC$(U&YloV1w}9eSC1dMNj_ z9um3CA*4XM%R0mzt)m_6we=E;&aCLRcP4*8+lT;F63}k?l~HDeGA@DKy%n}8x9zB9 zqQEY@o`NGE7ZaD0rMi}#8I*{?Ipmxa2adtRn^Htpv7v?D!Mev&L&^dIh*f-1INUEX*Frv4~!t8#=0;(ygaOE z<-vFZ`{GR`4TJR6?W$vG^5pidlwiqIjIgiT40N<#XTZS+!PblCqrT1u1B(&!6-j9; z7i&|kC3&wJU6xZwAm_+Wk%b`9CvSE(bu6z-1%7ddxi#s_9Nq{w=2eaM-EW=<%_cS9 zT^$~hMssL07A>^qf#-d`4ChgP4tS_NCa|{{{Sk-J(&%3J8uNl5y~Lh0K1Ma9m-%|0 zg~c?v>C&~cK3GvGrTIu$KKbUgl=_H%Whv`TgISNDCzizB_O4E!VzJ<5=ek(-6%*Ej zMaQlhMzj2YQ!v?SuD#H-t;wCauuzeUVYvO(`fnunD#9janryAUgPRJa`}2i?k7F-# zHt~oS1>!f6O{=wm8meFU-oM^FzmM43Rk)Q?KjzyBHQXV7w174ab={Ko(P_h-@cZHkc?$a4%P29^ zv^$%009$+4RcH)36+IWA{XN>7qZ#XCgs>u&@r@0jX$1gp(0#I8(1J34mG&(=Z^QZ_ zFK_&XxcLzd0by3B(`uu3H1>jQgQfML6Q@v39|EkG?-QqfNVBsqfn%{ueP}7!(H9U= z#WDM2E~A{uNQv=o#-!xn_8l1EQ%`Z@t22Ary?jq=HafcT_K5bPvgM7>*$86PSFd5e zWA^UryAcn{FU3{9*}2*nyU$#t4vX8r7vDMWKZpxIa7dkfijIti{xk-gYff$II=5=( zBX(I+F)h**_y{bN_zt&q%UP*(vV)4g*0^M!cy5X?-d>c=!JwNq-~@wV?o8H4$EP8x zcRnpD)5(pBXCqVlnktO0@YN7~?p!_CFUV2LZszd9{CqH(iO)|#XjtX32@0T!e> zH&`2T@xEbBgSb+ap4QLZ>|k4s+Y9RE{{2-?kvCKdo1|NOA4mCZKiwyU(o?i!XFX`1 zim1NMAr{_1^QWuw!|!A0flZJ-R_*t_`zTqTtw&TzRpt{vz;d{&r~to0Kq8 z;OO{!tE1`Ll~fK}u{X-uPOh zA}`aGX2PB7j)TJ}9z{gOpVx+p5Y^kgrfqcuf`;c8X;%$z?KMwo0weabk~&N0*)N3q zQX{jK9Up3PokI`FNasc48Y^q0MTQnn$u_ljIGTmld)!aXEoWog4|+a%FY&*sxcXUQ zUpzK4%JUD!nwu>024^_3Qpf5#{C0RYjXx6jeSp7y3*hq`ip!5)+z(5mG}&*5c0D~m z7$8aUv45o;79Msmu=@F)_KabE@cZ|Iykuw9TP-^Ez>mEqM!eG8xy;sBGg7>RuqaOy zjS{>Z#2oTW))%jgzLsd#8?GD961`1W)QN1$X;WDXgs-9~z7xFz-NI29Wv8k*&m5F4 zAXL4iAQFq2ciw0<&_^unxo9#vyudrX301D}XuF$BF3H4kZ*e5`=?>(^2z037UD0e= zJPtE_(BHwkHr7fEjE&KNmHx4-Xv=*4QxA13{`Vvy(3$)#RfTGsx!}!g%Jb)#)M&-T zSF8MLR@1rlTZ)HCV``E%yT&2slInH(lehb1X)M1XIOex>%%$(E?ti=6Uu(C{jz2&9eGwm{=r;Q=POM;GGXrvW4qs)y5=WXhYgs`^gHlsE&w^_kSw$;qHJmQcykNkrox zLY>15lq(}b|6bi36-RSdY*sbEBUm9|mz$hCPvvdoxczKI3gqA=tLNr40cw@P$cje* zlJf2jc60itUwr*4&eGl$=V`~zX82>*3%82}#*Nj4-()(a4u8%T*leENcr!2zsD<@` z#d_bt1d0+yEA+W>v*UyoSBY2fn+I>>OlHT$aWqI*kIOHboJ}-VQ@-j{58h>?XV!{@ z1n+EEcxiZPZcfTLABw?QDQ(aNNX)N%sMRK=NmX0Bj2IT6>g70L!z(1A&)AB~E8~$J zP(Mmi+QJo9pr@^FuW&Y(xb@dI>LdB!W{wqp`%AO_XQRqNL-FmFK)2`fHTfxikDWKmY(!)E?&HW%Q~W`usF_BEXDEF%-H{O1QQJSY@4 zK@i{KD0j_l)&x5zNUO#-?AGZ^qaoEd4!^n*J)b-@ui2)FTM+J+K0pF;T@cw5UH4l) zUs2c71wD};pXD_Duph4np<7h*g+)DAQ7$?Er~v<)#$1Q9R*s&I-b4TADgYL9O%{~9 z33<8|c{{Oob<|x=RWInPZf*5?bB*{X8)IAUGEc>4=2?ze>)!=XrDbG(<|j4^8rX_p zc~A7gq%W++(kn7jRqW!E|BInQ3fwjRDK9*~sP4wO=x981c2vf)F?mqF0pqj16qeh7 zapsn#qa*(+YE*udC#Y)0>+WG!xdY+?avplU3<$|pp*!DR=WHHC>=c_Z9Z=Sr^)#a* zdAqK79$#x@CymAh#`xzd7=704^R~v7Ix>d|3aGa)(r=9@v4z9_mho2A>$+>yV_z|s zKa=cTkV{y&$yoEmvhjv&+VsBS!X!>(tIA~n;mD$^Dl<*mbdoxY{r$abA$#{}aEh&Y zT8(X3XUb#a|6?90f`&YP#YBuboW-9~=$$jA^sX!jGO3Db#8FliSLzpQplnjq)HGY3 z;r#3Oe0zu8fLF^3S~0?{+X+9a%dj@F)H|*ISRCJUG^Rz!JvU=}lwZB|uO2^%i(F!e zHR4o(vL4)FshcX3_WlU2R{cYYDw0>@nV{)po{{bBcgURd`>qXzYU>4}V6w4u*UoLbtS`YPX``Q8J(=VrKO~n9+V`gIHNU3IGjJJJmlyg2CHKnl+z_gjewO zj=|{K8y~9mxeA?6ViO{kR5wO4Ef6Bir|m?aG*VRr^*78G$lDHOvU0d)B9{7u*2X=^ zX4^@&wp@<%XWdXY(l|6r8W;41YAHw30M6ij|0kW>Ns8ijYm={CF>)Rzok^C8sr;126<;nHeP??n4%r?syGp2jUXjxOvQi%iV< ziPa&+h&ajQ)+ONgpQYFF)ZSL_t?Nw~s()UU_-Owmh{cv5X_B1pV;MM%PpzDf4J*h7 z8Rcord3BENM&Wv71tz`M8$?vLns#M_RW@4*IbC)%#qaE@j2HEFwOK}fAkl@*LPVUV z4DD0D*HIJq&URNCd>Y+#CIQb7YU7FfS(6TD;WS|Buk-cy?ecidEEJbrV~(~4yEwsd zhc{qVtH8nM$s8O)R9o=n+nl=}ArmiHj%tHjyxf!>8Oot8$;(&(SQT1nud6}k!CuUF za}8k;`uSb5hV=R>UeNfg$Y`C~(Q01(S(2TSl@`j@T=%W)2==>-a>b#b;pw!f0?1&O zD&_Gvi0S4h$Kmu!0|%?u6%&RQlTKuVD#>%o4R#6of#4jb6phJj;<+5_xeAAbx?{0L zgq9o4`MC)0g)BE!zsSDHBFCoSiE7Nm{62$X#S0}I&hDmP#C^+aSYvYX#0xb3oF8fA zBXR$!kCq`POT${CwZ9B}DE2s4ScD}Z!6d#buroAHBVncl>NAxm2DHmJWgtWJZ1ddmzFS8_kYxr9QbN9;WUM4N<~IiCHFT* zrzO~ZgJH>pl$kr!{m-{)hVrH~oR#@oWiBZS@x%+V{4M)Db*jc_iS9(;mhc@HtM2$k zK9G&|xl;;q9=oPo&c$poe@WkGzAtM>r6rX$C-&Al-2~b~m(G0fwi+8|7o&;mVH(2MeqgUWmA}fU|VzuBe}b zSHRI=P_8U2qZH#pP6sme^0)Y#YJ!w!BL{aA4_`y&s$xY|M3MYw)L6KH zlmur?*2$*Gv3br{YOtw#)@^hWa@$GMaXkq_KE@b%9zKJRKslQ>A(YeY>RB>%n5@Kx z&-b4nd7;p~c!qu#qhq|G;#lYalPXs^HsZ{LSdiI)kiu@Lupx!+-=|N*tKeO_fU0C}j2-31%)k-DiTt0ia z;Pz=$$HGMkefy4=mS{Bg)Y=`=JuL)q2+^%9>tjV9pHc&PYMrOdP{|Qoa(B+{cPXsF z14m1TY&ds~u0&_G`)2E3c_%lwZf7l@tr;JJyKWngn5&S3U6(7tm=G!*j^*p?jC3U@j9WE_+B?$)z1+(3q5)!F6FrP(CU0TpMPT1t~+p@bZ$I6!QTDB zOkwo$_gCC&+RVC$$CTbi;e3$!u=1;+SKwwASqjk$;Fli$A#W3XJP}UETe#7r3Dq#smur|;7ukKfe>Ep;mrG25tDz8?+tEBgarOF!D3=o>Fyv|I zyTR0=VxyL-YRqGqq_~YnMElOKjZeVG-jY{MV2=k<6hCHCn}BakRAy@w@2XJEykQETxsf=BpN5DR+O<&av?a||H=(>^K|^7lKg zQ!4}43yLba2g>%r6@_Lew?f<@2M#6v8EG7ZdPAQwrk*w*)?YdEhL7Datc^}C5YM?5 z6vX$Z>_(QLLx0NYJxL{B1xm2{{WyAh*oF$mfShOF(N_RxHa;&KdKeMVHoKi!K2p?o zC@@iiC0Qopc;(up*(dLHXO7S+UEjqmZt!S&66hhQ-FHoq+17v?=+`h zZUsL%oZ#z`TKHeIePvf2TetKHfndSiEd+<)?hqV;y99T4m*DO$!ENL2?i$#*ySwY# zInQ&)y?@~S#D~ooySw-5wW?~?tXY||a!oB-BrcF%zi0D&u?6Y zRjyBUH_e;Vmt5!PwrN?pV`_aLnG&xZil5@h&a6J`ugN}j(OBHP<`F>ldu`thN9Jgz2J(m*W4`Fe_yGe5X7dxfpEmVTEunM~`C?jaox4LA=K7qnJZ zs-x^J%`9ekz4RJ>L!az+^c!IhS4`X8=X7}O>n$xFl;r_$zCBKX`YvzvTW)lg=1kC4Daew5d+; zNYH6N{Y*PT?NfY!gvgqb2`2kb2ghH~@q-4^k~*jMYKFDlna<;Y;q6LY()w%TNdu=> zAnU$0=#yD=$YT22HF&QvUq85kTTygr^LKvR&NagO8^9%8BD}uK*TP*V=BZjsy>b`L zGZcW9MV!l{d}X<7$}Bv6KrfmN&v+$|d>}E(CVka8OuAV`u_drIpphg_#geL3#tD{I0+$FCXcxD@n3lcvCN|axOWXXO zS*ib`L^4m!dQHa+?C-sr-QQHiTa?2g|Mx`ofndX!;-U~m!Lr9*XS_RyLrIF^O-bIP z%^~lHw9xyrkkf^1h4#~b#u|G`L@4|^3UmM90HI$LJsFe7{%%>rqIfP}(Wp>SK7)#; z(9J%&EFq78`Gya)PRI11pGPXt5$~Hd_pWj{-7u|PSCHwRT&`Pd(+wObcjmk&s$_(7 z5Mr>V?KE0_%`(~}8&};fF5;)Dx}J54T15TcFE+c(@$OnwUOWb81Zb_g)cuvNrdG6| z4ZZ6P0V}!R-*)BZI*zKJ;Nj0osC~4TqOa<87v7U(tJ~|{LZtqwpt98Q=e9V79G6Dxn|rTv^KtiD%|ID85B zlh{TiTZcshmT!Ir$U79jjJhMrx(m|an zY){Qw&>B9gWP4RT9TckrX(|IVcnZJXHh*~VCYzkn=Gs%yg?!p{s zD=l|m=f08v+Bi?cD2uivDTCyzr{m!Dj%Q}=qgAuK3CIJadW?d)NL1xv(&<(0VjNIqHFFnpM*cTn<-p3OwN8MXVL^T zcCx8$la#1mAzbw1_t`KsVhqs^@*UjZt`Py->rca7%`&g9Q)iSweGXA3t5`;8?o2bE zR{|fzjnMl{D>;HcD2&=E4;#t;Kd?fU@dLlqr4nn5Vk@@jG$NO)aH~#=U>)4@)RvIN zdHBVKINdA_J&h71=qB4NFLUqZ^<({b6^2XMDrXvW;}oBTv4Ddzv(w}}BmI@M6p<#c z2Ag?pTbs1DCg&OKY3c7KRONZ`S*0HWG0s|S<18r61ya<>U#JJGeL{RQgef>&=7uS0)7w%hVfCoMp9R-2BaI?k{@;AX5(P~*eb-(}Pc$GKUEe~}dz7~Qb|9$<-= zb+)Yzwo8W&{Wws~+FuN8du?3rQoxG@2UHo8PfLAks;(yJw=CQuP z%o!HdPXF!i{?D)U|Dj!1Kp1t3>(Wpw8D0*IFf~le_4=hyy{80Mfb|(Do4ANy;Vj*Q z((atBlF-4T%VtOx>{J}YbrsLKUjMUjh<_kKo^(WJ2IdG$R|3U4wZ`}0X22xxE_0C5 zEA?sZajkUUTl#e?tlNb`%*6w!L}*9-DB1J@U4{eeKJGo z*#ZG5Q)JxIDC1y{&c=^3z$W7)`!lk=t$H)GdK}FOZVYzc{;;ccsqy4({@mJb?cnKG z{ZTh@i+UNr89`A%*;P0o?NF&dX%ob!_Sz;j&sCsx^jLdqgU?aJ%xu#sMfGpi(0|A? z392bJ1XmYBmCNx@E+Yf4C4G|-u1wj~39&McWEBd~M~Gg39(+3rJA8cXy-QCt*5@sM z4=2a$>qSy7`N_%2_2=$ibFLlBcYcT%lRPbxV|EcF9Bnxd#n19!#!Xm|#utlkMk>1_ zzYx)97gkQA8|~ZwxJ7ckO?kyA@0N8$;UxqtW>!JABdTUMTJ4>cgY1iSId=#%tT`tBF46=^RHde4RS&Hk+EG@1&u3|I3!jsVCH+$s;=*H!_d^c%KmR zQ_?oMS^vB%I@XP!h?lW+Jr0LnTl{T=L=fUtN4Xz7JK4E(qkBALQ5y^Yyk7b(6uPsi zt(b)$D4+w%vRuRoWoj#f$Dj$`+5`9kTJ20=jF;4!`p>{g>H4+tA$^dz=+ovnRV@*wAmV(&#k<8EcG^Up@F zu*Sj8sa0wjhC;6JD2RF@G$iIAUGBNI@cK{BGjM_yiv?yvK0u2%jViYRwbO`gvbdFN z*1f@r$8~`CjQf*mG@4dM?L54%kClJkJ=M0)jH8n<<;B>r7?GL% zEc3zK*?X^1a~r0bjAnecQ= zMi>ec$(P_ld^$8i2>L|v-TC4k>0o{DCbdjg>D&udO#H-p+_6~s7y}(SckOnqalH8F zDaGlegnHxq;2KD*kJZLtk@7du=aC4Q$hd_>M*yaDP9gwmaJr(c>F@+6p!u6G{15q> zXPS9xzZ~5}BCj(o`tEFE)qFlSpPSx4dP3gW`WsKr-pJjF(v zLc?Nxuwq@N`I$0MNbacdgOb5XNv+%;?F%ot)h>!|$Nuu0)!>k8|Jgb4l7)thz(9)% zde$F;02sWjm0#zLDy|?UU{OeJB${>@f@qSMgaNQJcNM|!;(E2ulb+g4TorE1V|gCh zM{nkR!uDZOyq(n#)oY=>1&XR?sn3A^HY}ai>a#r4I&iP57c}@137s@7O+xDTzX+qWi`C-|#(tWohQGpDWbySSxQ&?Y9^HF<+DR0@p$3Ivvk}=YJhSLv*0n zzR8?v%9L9P`L6F%xRBun`6j8&vd|G30-9n=o9D&(7n;PFR;BZC;k=!gB*_^#er@V6 z!nrcA43&u<`%=84dJa!;u%>QSyYh1dSK}932+&ChiOE!$xuxHwvoDh(5{)B*sVQD@^*5t^L$aT2fQbhcw+pQI~m8f@H;7iMYH}1>I#$8C5y#8tP+I(;@1oa z2!un5OR6q|d-_K&?>Lv@pD?D(sAN1%ft;HC;*`G+{Qcv2n8hrX?BQ$C6=_1v_E_M} zt-qFDTT4pxL#rxrG|162@HZICrS8#N5-!7jhEeF)#bvib*Pt_^lHn=sMQ@JWmLu@S>M^lnXXx8t7)v|ieu0Hu`b(w7J(1$C5u){5(p{-GY91H+$uiY*P2$*j zYEpFtdz-tvkB^#r!)AwF6VRRIg-4H=i>!FE!GPkP41fX_czCT$eHQjUL0RLGw^H_ zan#SUY@VJm#^~N#RPe*%%$r}tV^nqIH;k29a#hBRS4A5fTn$w#vS z{lcBiJE5%s%bcN9#DiBH?YEV4lQKOyD;eYVRacr<3Q?wem{>Z)3 zuUwib-+OHP6_ghDhg`9Y9xivDnq`-JDMM%+V~q=(0tG7sST4yq~se zqED3znzIY^YXUs#aLxRJ`MS_XTK9-i_ed+#9J+;jVC=_5=#S$+j%=E}cquF@k%0Uc z>ujhxn384BU#4>OJ2I&o7ESq9N4aCrEUOCZGcLWK@pzi3n3$M|h--gCBm)BSF z5s$l*lFAmK7p3z_H*#%IfC>mg_yMDI9XefOG`g{|;p^*r#6xb#W4Z8+w-V?uYXXw0 zgZ%yNvcC8Qk>p@R(-@T_u-C2DRIx|tUs$+wOVCWn`grrB+0^V{N}hSx4fKMjkY8! zE0fw$2C!1iq4xS08!W^okY<_SThMxU-ZLPQC;PE=42?|Ax{HJw+*g1K4GVcs9G!D4 zne6Yqw;GOsva<=U1mMhBI@4f#ip^OSY8gPnPvenC)C)8;U7$bJG)Ql?Dnx{zwfPvj zI*-L)+WuH~80>kv)LL!f-XxNiP~G*7kXhrD*e)nZ&GVshKBly_QX?f5qUO0zVPV0SogWLd?eI3`CyLX}qAr7DhM9 zff@gj@eLt?&f)j$_FCV{t%KZ$zEuR+xQ(;5EtTRKOzMOdc&B^s0y-y`KuiruGpTTN zmta&DS7Kx`A#n&2*mWP~Dz~<_Uu`fVXKy_Q&dxg%Zo?;wttMx6K<^vSiQ&$-Ia}(R zp<)18`pM6Daq;fK33+R;zC~nrl#O`g1^HHp3Oa;gC*oB6ZSgAeYudNw{ddAZP22_< zO4AhUpZCnV)IUkzxT}W7`$zw@4u^HA8n6D6hHxJj;=OJiEQvVMXnY4>VZ~gSw+{v_ zh`v<)gF?Y_ndRix_j%&c{DigP1}{kl6qPDv%E{3@P9EJF$HsR(4~DFp;K&*Xlkn!# zeWH~^2~mjJ$FKc7X1Qc$hW8ocDcmB1HMVlRqkk%pn`38h_rS}hK#k9*zb$+%Zf+L$Ttf$HdFN=mVu4CIT}_vF;Mv8Nv4$Ot|lve(QdS_ zLyS4Qt|)^1Vv*4Iq+te9$$G6bi4%6NG6xT;a91#;PBJ4)I}CdJmm>AS-*VrJ40U6L^FK_slhqOmGX>R z-;z_-JG+_RsmwNpbjD}*sMbexQ?FaDTdvUR)tiuzBKIt#NH0ZbTP4lH)>5T-{Tu`t z{Zy+V@7zZvqc;Ab(vV6ZKfNa&K0Mekb~5ujfX<35bD{=n^)P(kofc<|+pLdsI=Fzb z==!8#q?|JJN~~*I#;s94>g=w5#SFxinup(>5TC5ED8Bn;QB|8tr}XY%nv8`7J!}kQ zx;+r7(qh8T--$@T^(!#YE;;uyAt51=M)fNNMLL1J3$UQkM@L5kNEx(SSZHWQ+e*>? zHi9-g?7zJ}i$&rZISoh6{1Fs^b~;m$;Zu_EDIHHL;B+`w7_A1Vsmp- zntP05h`D@^{DjN>=Acj}-T7o;I$r{9=Vw8^(vh+RWa-%PC(@DMq*aDvM%~=6uGE?y zDTv!OEJn5T#{}C^CuFpl<7J6FXmAq^R73LA&6)Z-P^25H>R=B^DzV*VVOCFNu!j~W zbXkl%wO-Y9NkFE!ul>tKB>^b ztNaW_eW|)TBO0hv7DcQ2_6k1Qqc%+mb5-xpL_D&$tb}V*1Gk9o=9hiu#*9lF5cZL% z)TloKqSc4KWn_Z(3sO>2`uarpes>OiqwjdoFWtao0^@RIU2<(}2btWTE+@0{qKnJ$ zl9w=XQ#bxtZ^0+-DL@f|q`1bhK`rDa!Sj?}v{@G@g zx*_0{#Jww1pvtN5E^7-Sqdb(xk37-}GFu{wctpwD9Xw!jIF*NyfBCYJQK4L>VrXax zbTr>8OJ%hK{iqAfkI+bj(sJxQLn+}J64SxlCD;C}vZe>H!dmU6fP`UdUBSRT z&rj*&kuM5dClMc zJLd;P9-b}L)94$GI!`h6WT6PythQh@Z^n=a-Bhx$YW3t^4 zC@XSw(4V$G49irW;MURi&4hvRYe4%(MGvE70(r&y$t)avMxnIhT3_=U4nMc$l7Kuwvv-<4RVT<{^5Z0X51`314m5FkG;v)DZMP91B1Y+0cbw6S zz}G+jlO3+HHuNz-24dV;!7{yG|6ICtIT3Cj@`DEjX4#;hfey87VQA&{Cj z5xV(D0xTc^;|PApqE1XCS=;o&)$QOimSs$*RiO^(Pl+QHc-R-U4jT%9UX!mTDvKWk zKb?&XM>pY7v-=1V+OctLQJ}M=q!+5NEC2EzxQ_^5di$7+H2S8t9~Gd^@I*2ZRPZI> zYqQPa5MDX_m+xYm&-ia7xPB}iZf^2~TslD5d4rXv(rlp=tPNVbh^VMw>uFUBI&v!L z+fOS?_TTM^s%5-$m%WvC?C+A7s;n6mYsEWbI1n63 zmqm&l5>@WrLjKsPk3yLo8d!y=s$~cDguq)ZA||GZBja7E(?0aYkT6^1l$eRdW;kb6 zg?zW}i3E7J;=GG8k8bgn^l!?o(B>DGMyEy&-ef8fZxI_uc5c?K_}7=VMLk9N%W+j7 z%wZiC&SdM z_ctPl)1wo4>HU*2NhJ?c7RWTOJj!AEqU6SAvZ>J!8cVDKHH=f?Yi?^qE9?9u~Xvt=-Hp-~E zt;&vc3_y}=c+6xdKr;BAvu`yww0lVGPmt#U~@ zF}t(3_ub-nzLEknCb3?cIH?v+5lswg;Jx}tM|P41LMUgF=0((n@I*P76AMabS`@Cr zwsp%}(bk(wl=x`L8zDTFEsI`l=Y2)!yOU?gcS=9xA84jRTGzb<^Ygqv&V6(Yy_$cF zBXe9dY!zuCJ@f&SZx#Y#@Q^x8w5D6!GI(17R$XCwrU^tN%qcVAGSX z{BaE^mL+3D(5f*E9e=T(DE}v9WU)tRyuM;JpiNlp4!uPo1_fmnm2auk*`?;jS|wHx z6k5pdaNHN6Dtw8TBlQ6LVkH`UI*Dtf}m4;6%s$W=nQ06#@Y>R9R6)0(bvK2tReNE~A5E-dF33VspV!J4o8;trzhl5Rz09xO=O3BDoy%XHtiCVPmF!(U+5SL^;JTlB=*Tq@Ql zI(k1QJ`oXY9({Jz_uzKVK^ZK2`z)OEd-Y_wp4l!hkIZAF1zVeP5U-UFXGa;&hq+_t z3`O{T;h`Vn^o&%8FY?>kn@mMZ)pS8R5@;o>szG&!Jh`qT@l+#~uw+!a!~XnamNvkH z5%Xn6C`Zl*3FE;Bha#y&u}B|3940k6Mbl9rx@V8H_6-Q3hufEul>v6zy4%Jvk;BcS zz}Ehwej-I`c+>K=?#;R`K}}a-zBO6<@J4rL(_3c+7cC{1YiH?424{(_X=WkWy+(oB zBJF@8t9QN4-jr?}9ke|!w#DzMgH)Qfoovr+WtoU>z2fjHAAf%{_&HEr^H^v-`OpHL zc~-RX4v1`|2q#)LU2O!Nm^+e^$dq&t9IhPY=9*TERzZ#1SYUtbS*OPc|kSzS>P+&Lqfu zlW(v`COFApb2FmJ{euioMtj3RrI~UW-m)Gm!r?!c+O3@%GVpA{PqUh>K72e`g>Nf2 z7$0$#wm$J}g*(fT?FZ3E3n#17NYny}6Nm5e;Z^iB_v2frpEc)6U;AtJBOlo8GiV?s zd8<<{*$v|p9B#Fh+r57YV0oI@s_XD=`RN@QTAXFib+q$(uhL-2t!1N*Tje#_2;(D- z^|wt3|1;05|KMvp|4mUbd%3Ifn12DaF^nZ#w&LvKA8#xr-|CXqujSroP*BVa-ddht zLWnl{hM@D%=CID_k2aWQ(i)uH6^yQ)?%kYRrSA9DX^EUn-JV<8tF3IV=FU|*%nzT& z*BO^xxjmV5-v_1SikXYrWWDM-r|0KsbhgktxXZyZAF@caR%0*L^PSG8#EaZt#hmax zh@DhlyNo&X242IL_k7Jh*^t=vZcuJHm8MGg;~_z|{QLp0OpI99MsEgy!eR)RU(f6V zk73yEJxA>2TAx(r7}g&5>xw7aS6|LgZmBz#o0lo(Z`r>f>}~NFBb7dQ1QDlBP427M z?*y26kYpzK@%HMPQB2ih8r@~9U~f;(RgS*!`cq+ga9u!kEH4%DE~==c7A?(pL5SIB z;AdP4#B(?r5@iQ7Z;$C4`_pe6NP~7Zx{R&&JL?&*HRjJISe9w)K1P4QGf4jAj>F6Y zJ@Yl2k!tyRGJ!Be_E?TT8u_ay?}f0L$FSjLL5JbomZR>KN+wmdzc29*FiK3M5?n`j zgOdlzbgv7y#c^LR&Lzp3(M=T8 zvJwz}L8Cac;?>#8>Wg!RM5_11t1Dl{81Yc8e7__`{R2xTmfl?+$(Cp90##VmGcb&R z^4nW%nBwi6{uY`E#o>JVT!9WAE0OKQ#d$v%B4X=_KKl10V1BU~6ZZXWWsHD8AQ))I z$!3LB!8-8H<#!Tqa-Y0KPTuIOJakE-9)Z1 z_FUzy^5U}t%W}+Wty-!aqE^qKoMFt+_~@YWZ0Djzo;}@; zhpw+R$~#PNkAj(Iuu2{A@1m-J34w0eNWZna-v9EI92bJu;W#=!b?Z@N;&L;05E*8R z)$jSL1+8QqODbDxSaQ}o_cljn^)U|9S;q%#Trs>3CzW3?M9!s z=(6=3R+Y&*r1Mply7mq?UI#_D>wI z@ARoXql#y?Ew(SQDK1z%YWDRkDVn9#*VC|M6h=*b6^1G*2N<6#g5KwER4Nc}ZPTEr)OBG@g| zzaq=qa^hLhztTWzJds$nP8TB`3R9w`r{nzFG? zi}5CYRioZA&RCJY;>Fduxw$Ee+BN(MdkQ#PQa^R?w@gG|4n&JVm5LB(RXo?cczSl3 zwEg30^7fjP8ixsJE2Sa6$Ory52=(}U#%DxLL_o`5JecAM_YYTlTTuVT&bQm^;V-Uc z$IEK5GQZ^Mb~c=QL?#PPVUKTGrOp^L%=zKqEpUVDww1VLA9mgh!I|E#VmRr8SG?~r z(E&W{4A2C%<4paAN8bqMoxGK7{Bv{}+*|P`KjVkHysO*am6vS@U){p^`yYqMaw+oB z&_QS$e7!ewJOUyvFa>i^LyN$>6i_QaA#?Q$$c!QIVr^o~Mg7c2M>f^T zW8@^ZW;@=xHv)?o{-E(rsfqFU?LEN#e10eKO{uO^gX1>_@w??Cq2xHNq?0(s=)+WE zrjlDzDgZ`41cv&CYNjm)xnX?d0-p!#Y!#&DIWewMRc{(wKK>M_jTIM3Ih#Ax&Edsu zVw)01vS9r*4lDIJ)L+bPwK54~28DA+*d+NZD{6M<2=V@$w+#$~kt=n8n2<~5${dc! zh`|p1Wa(ujGlmbtLZ(lb6C1A^4k7(uT9eApVBrB>bYV$fvBHfO$Xc z-{|oArcS}+YA*niA1(5m4(i7|Bdk~p5spYI>z5BC3@w9NV+t7w{QJY5lku%y^dOo$ z6J#0hiPT~GSULum?ck&cYq~d_w)`Ho)wbh%YF^-m;Z7X#rCslmp*knn6$d z_TUYEpEBTK+dxYiI52~Jhx0Igz194_LpE6LwGNh&j$=ln6yW%y z{A?VbLr{3T6XE)&T;#=KM)k{BUh&x&#BO@m=Q+VG)9H?QCN@N;fkX%_2B>R#gLwsQ z;Mkx>{~&dzGT|3)| zQ5$8xCXw0SlgTOii^KK2bbQ`oZAHOit*mf3D>MB{6TLux0NJUTupj;D7^Q>?m`;Y(LPF^8iA4%f82AoW9kJXRH9TfjDGzm-+{)^Umu|juT$Ctt z7c!GoKd4lSKN7!O!Oi941mc$=y1%7zzwTgA#PZo$R<#v+Jp}q#zSxhyHH&{5?t}N= z;+!p_M|oJ+yg=>|OkX`L9C?jTu3d;0VbGTw9XrX36k4*qe{#J<>v!qHr29bld-HJ; z^#rBsc_tD%c_V+il>&c6S60ar-d-8FK*$ehAL@usy#EwB%e9S;RYOY6P4A9F#Qpt5 zFh`OYpy(SGqytclC&+8Df$rzg#SG&XZTSX%#MY{gPBixuLgiz}qCE>J;U5L@(%|Ou z^EP`~ne4_jRh^nT)zv7(b_F85&JZkfL~W-mxT45UK20B7H03<$$-6PGPFKzeA6GeQ z2o*v3JSJa-T|FOphL;U23u$M9YSq51byBh6!D|seN!`C#fl-Q7pasutyN;zJ_JivM zkxl8+R^lzoXJdu`-iBWsSyw{w$1llkVF~E}UV9D6hF)_=ccC4U3~N%rY|KY>(YL3z zcrH^EKUSSiM{0MY-1Sz@*4{)q-LCE;*oA~RoJmv^6=aQ$QnDj_SyLfC^{pVbs2_P% zRu0WT$8u7j#7u|xd&m)PefA=2J#jyqu4FDnb%nig@NXh4N8bS zW;I`HV0<%=RUR&4NL46(vQaN7N-pKQxT?|RnKB^`eHu4k=krQ^0qPEE9W$(YjR%i6 zspKAC#VV;dwJ&DhgXk~*z7CF5<5?2Djxsf)C5a-FH4>P|YU=tLbYV0g<%q8 zc7D9IzBO^R`}#q|$-wU{@b5wZ{!p?4%QZXzacM+SFaf(CqttC4hO_2y!5G`|?-M_ZJZT zyZ%UjB(nYYdnJE$O=0GR;ym^6&!-kJSU9vIx``kjv%iHsnRic=lnXW255mJczZlb> z8)u^nN%CI~?(StR+g<`nV954Ih}8~yybk9Fqt|ES+T9eGt8&(+ey6^%h9%)@{MMMl zze_%jY(s7M6z%%Ik7a@kf|hAcGfD5`aVfzCRJL1!t$`9I3);x_SZTxLK@`S9KEEKIkkn^Me?qBX_S<>Ep$ zlDJ+cAP}Ld5-Nn3rCeM9$`H=t;dQQ0D%R_9la%*zdfVxB!Q_-i@I?oJ7k1+N<#M~5 zDrU&~Mk(ki6e2?{2GE8lO1aAv@h68bP?5u(F}!HJ1(28k0m$vD z<(@P^!^P}d%thXU@XaejGOv$Os|OeOc>m~R;%w`rO~!Y+7IA6@ygx_C@Ex)t->c{G zy>5?g^)H=txp5C<`FShm9n zm6sW(@${CChd^?7br~B=+KrLb*)M^68+(ky@n9998V^w&dx>MsMGUCB2-}`{-WY*Y zWR(@Lc+z=?o>rBMHrrgqDc&?_bAx2u=D};bs9!TQ4d+dJo`H$9*vRwA+n+44Z}Rlu z;Md|yIe!&0C-g`Rj;6Qtu;j`74)qNopsM)B-dix~J0?@;WO445e>|Df6F2-(2EcCK zr1Q~x#ah#SqINvphTSRZ5DrGU!8@$W=4UjB>n)a* zWW;21UCoK_&C~Mj6k9hDzn|20tteR^R0e?+D4^=cAR+zlCcn`}oD<-B#DUEPVowz+ zJlRJcqM5BnMX=k6*r#dW*8Ky`m*oumy(M^mfKLVxec&2&| z?fz(X7)!pI?S&8H-zQithiYgKGdc|~4lcm2w1U4{!-Nnu>mD|Nyc4ARC<)U~G(4L3m&@1r2g<$OZ0{f5s zKk{jRv4p)?*UStvF>%-6V9`U8lv>1hAdy$I&UABsUo7o;C5zVsn5LVYZb6B>LmAiQ~TA79_D${0mCC_0@80;$ZI5;?i!93VMI z7g*(9p4Ly|HZDF3JdR0;A*MB6D-foT#?n7W1=S13KQN7na{Eva!mu4*n#Vu%&AhDn zL#q>o7UHV1S99p3I-!vtQXcoo*QR`dLx#c5?IBW-7lyZ&jI)nc^2RD%`JDI?J;JO* zrPPoI%u)x+OJjgbV`RkM#by7o4e;0b0P}J#w?(PLsh5%avJ}VwP%}FvAu5=YMs@*& z!+PE=ngFAyZV=d4Y!QQw4h~={12@vv#L8qzIv##q;9%{d{ZX~zN*v~Ky&%@v2j&iK zDsR17ii?Y9aygxlO=XQII>aO@n{b}i0WFtQ6moeBrF%yrE9xVvX`>uQGIMH5xCXJr zMHmVsc(ZKl@>ZN_wFBO4@cYaLy^|oM_>lA>@vpMFb#LoTu3Bfq_@f6npDD;`y32pe z=FndMURuOd9&6$HNgf)Mril?V0|BY!pI{R1KNFX~f>%P5yClaY^*rVZuXglKwAH@R$eqLoAK#*7i@JGNWqL0_`3pkfu_PNS^Hi!Y6*& zwA>8O1QV%V{_$>bgbcM;V(glC`liR7btM-_xkGz)vok`P6Vi`5u*}G7SyZyf^S@$R6XZ z%KWC`k|HVHS~c1zTaC-*>@%J5(oa5M!ge^g?H6O~6_OQ3E!^bY7`3fw>q$b7GWv(b zk=FlO4XR-xi1_)qxSp0QDpvt33lN^mYd91GjEs5&yd?8l4hR|{^SbP7&6=4>OT=di zF$P*3OI9E88qDFEq1N;>X6)?jJaZ%G+&|Z*bawKeJRZnzpK^kqZw?J~YEog~nDB>h zOs~!DQYSh}$3J7RExCeL?CMD-VGb&}%70jImLBep)K}QUGNM{L$+5<9Vck_|^+wE` zP;YzFBYtRBxkg^VjQ8HQu;735a#7G&Oo08Jr#$*nH0~}}QmBjPgB7`^5IIm*#rR2T zuh^sWayfkGeigC%2TT|%$7#P_gK^|QC4?P>%V;iLn63PkG`?otUdOG3lkB!>Y3{r<6Jy_6@|QK6v{ z($YI^^xP&spm9}4V;&h z^O}1h1+cdUm9|e(&9-=BVb?6IkWt@gBznv?EfT#I<)HAG#M+eA6;Q9LwSEmI{(*tQ z5$j;T6esLc8G^UgJ~#B{B()vsTv@5{lH(ku9L%Sj$&TlP*p_i=S`P84?BQIwonm$Z z8llng#KUA`8$6%~h7L*7KiD|s8sV$1!nd8y6&+*#`04XTrgGl*NPUBFt>LK(ZSJvN z)p0|kyE|`^DexgY2-UzZi}WpELf=Y6q%TJ>Ks*QgyD*Q@ z2s#5p<;{&_*^ELo{A6`wBRh2zFmT}VdtH?Q2MNgs4|8y7AW_JC-pV9Lph-F;t{w9U z&PQY}1MF(WUzWj= z_)x~>wj@7zC8%Zn0AfdE6K}xx7Z^qki9X~#h%F$e@i>z-!+kwZ`^A1{uAHW`=sY+) zT6}<7jGvL{!=G5MNU&bHMNu3wue8I_OF_V2onzGaRC>F9G^9{}-tRw#T6B`B=GdEr zHVRm2krQJlwLbXKf2I5Z*y>(Xh)0ZW4-?W!!{tno;+ysMpFD$h_o3Kj5DIxtuf6?di-$`H+_#tQy znUPkXfZ>EbqjZfM7J}hCCKLE&igdK}fdu7-z%WLx8n^}KC%79D6?PJ9*(wt*AmE?I zMjRRu-&C})dgcTxNxJ0%6vYH&{r5y7^y)O(ZpM-ge6n{YqyUGC)bHYgy?@(-^0qpZ zK}9ssVoszIpu?O$s@UG9vNK5;=5y1U)rMhtGH*-W0okPl_X~>#k(3hy8#2P&8VHF9 zjiGS!U-DO94yclHDe}+P6VZ)oBMg@U1yYNdd2yDfGL)VNY1Va0 z4^7OEO~K}2EyUwPCc}D9=80W-d&n5?27byNs|jMi?3OtB=XR1}sV|fL%gDtSxvQ)G z1gL5zA*7>)&n04HDO2L(KyaEk)j@tw@^aX$+DOn-EsgSjL^)q2r-?BJz2-y?i?uh}isc_b2KK?3# zij^emRaB{H&5z|Hk1D&PZ9-R*O3&zy{{DoqK7!EW#aBachmT_I336gh<-lLDqqyy;9QUlSpR z9pjSx^)cMRl}r48{Wf(Vc`;QxpHM`yT1bAlg#o%^NEdqXiEFx1#&|K)wZCg z)$6{&*9DXWipd$VjUR~@we5aKvZ@W2KF!j!zgC;J!f2*!4nm>s{gmZ?&ClBOx>}ov z)_07E^l{Ud#o24T{slU(9f~-y#6J>|^#K174re_g$9TI5;6C~+&`Af*Zo?M!668S! zLXk})8U<-}9{j)l(h$90QQ%8!3CP#@gP1k9)<3Ho-#4E#vQ znf)g7BLYIjkYWI-2;KAP|F!p*0bO<57bvU<2uKUkB_Z7*-HoJxAl)I|Esb z8B*&nZj(#F!NH;avBlG4l|%JcXM1+%H*?IIai#ijIb7~X0B`vom%Tp_!%Y}K*F76d z=7?CqUhq8TT@RizS=pa$a{KNmYPX!kZh;G(B;C2)dVq8?8E?;Y_@OT61YIFj3}LN~`X@1p{#9*R>%rMZUIhX9ZrGTjDcQcuNZ)q6m1qIu;xKUtjDITGK$rrG?V^ z>gpN+&H~r^723sp=|Bg@JT;Uzvn*}rVJ9_)_JH&N9aopTF89`Oy7pToTQfL7Xqxw6 z(RE-zQ?vcidB{}TZX~Q8)Um0X@z^uj78~HHl=PZTiauee%<@&5Xfv$K2NHzo7bfUv z6Sj5>QexHa!=c(!Emn3Ws?`ns9Y6QV-f3?gAz4-h8K~I(6-Rui(HLZkUX6VPmVb_@ z>@GYMKu5%IDKDY#c!jo~ciUNNAs-Q4ZHQmu$5bxn!uRn1m>{a7^Q3b{NroX)`omyX zLdZ_qZAneQA3#tjf;b3EB6u{iJbK?#!4%a6)hfL2ZhFlC`q<^(Q*k$cye_Ou4-t|) z1>|T3EtnAc7p{t`sQFTGBBGekok%ZK?3U@3g7Ot37REvqITUR$kqm~EXs4yZWq#^< zPQ^J>d$@jegnZ9bsBGB!n0myTrg@d+%NR=DS`PRPr zm8OjD7+| zQ~7^b>476nu|M?DXEvy3Tpzbme9zW=VO4};hMrQ4`9ruH zPiT>Lh3gj5ioehPcLW*e5NRd42aaAaIO@-H{rO8^69V2hAHzF^{ySL6Qj_un4w2Nk zVGz}yW&58qVu`?dj(y}tfAPP>KZvNIfzOI&G@c~)pO=P%X7W1^;l~7DN6Ith9tlh? z_4>)GGCg?d_^#|Hn6_aEO@j4Pbh>X8W&W(}Ow>q5M^O?bew0q!*p4h}#DI>3h!iy#d9$;rvN*x(cq z9)59pn#5u}lqniQt%vP)c}T!&GSU-8`sQ8Id$a@TWXO`hL9>QesM{g1>~W^r7W}CL4fC|T>`igEy!v$vc}GA-qxzZXnm631!$}yM<}CW zI(wo%0H;*9%eb~K9SOV7x!QER>xaMa?!u8rhZYYzZAGojPzhR4aB$^Nw4EE+)GScc zW1ByH`b5iqu{#+u3|bW-R8If==^qE^S=7d{V#r%Fxl8Mt%bL2Be|jHlo9n&3XH^#n0DrpY03vRX^#nim9~nO;R6RLc~OJ0xn&CSg})<6#~ulwJsDYIbjf^&Mo+VzNoKHpMGgqfIX&Go183CwVJDj&x&Kv<};t7rGbAw zhKzmY1M{MK5~Nqk@m+89#nQ;7@wlA)#Da8;1K4U7^!uqA=7S{q05T=`{;em*O)iIfw}fF&v4vvFWntx3qP%nVa_Bdap%ln)T)FGpW7Qre57 zEOd$#sLozhe}1FRW$PkyO5W2-8feM}nRcI7`~|IT)Hn2-+qSz*8A1cmVl-?}b>8$p zwle|$hiPT__;4SHh@;q!Rf3F+e0X?BPOIVGVq1eiLQJ?#eQucg^L=9vWXyEm7fu(Q zmYD=U#1gU?4S>c(Y&L5mzhLO^A8wDjBZyj71MqZu!T&g~`eOlzE_&J>hIa`%fO%bS z#uvZqOh-};FleZ+2My+Fc{R=Pem=8+DM>+q0(?Nsu1MxLE6bJ{ zH)9VJ4wD1-f2)DklMGA4S`dIsFR`en#_#_!2Q>Q{J9TD9NqH8L_XzC(9s%Z3TZr2{oSzuGrL~Gh6uLKMud}ETT7iHL7ga zn$U-uf{>nfN=3=IT3YVIF5)!Mqr6W&iKTK%7W?Uduo-Wxn`Fi>(H~b-weWUCnox-4!|A^}O_NedY z_woiB?RcQUShGii;WJR_vC$ss#fxz$#2LnD(L;M992IEL)zOZPhQ=Ip7*v_0>MDgZ z(jD(M_$86)3ZZpfy;_VzOF>6RKp)+M9&uO6TWC15u<^<47JXa5`UD^oxQgwJzu7mD zD2by3i3K&g_GwJ!jK34|{*hcjcJQI_CsOv}a{5hFo`y$5)aHS~x3ZINBINIs)aEhi zw^sTmy}yS#R<2LZ$x`cQ6>_VvxqhV5=(+eQ>-``U{)HZfh2m*UtzPT9Bvv55M&7_a z43UYNf6X0b6T&Lq_Kt;{8}IaBb#=5r=?%NqNQK@Ylu`TqJRTEk9!4%Vi#GZsHZZi4 zSN%0i)ybAW;3H1K`U1@-We;qdL8nO2t%~$QR~L<>VmE={;qFY%$F}?lj%N2tn=i@8 zRo<6TCY{zNi?2h-e%S9ot^Uf14dWw|{?`qn3>DZmnY?qWS=UlNqEddOie*)vtXFcW zQH0|;<~XWTnrza(zT_3nIAacNWjNlS`(8jwb6_}ZA1)Rpr@;?+;5ARx57}rhU&bUP z^t>6(cgB$xpe|go8X6p&tnKdZez_8>Wa-tIU@?)epbHhYt@hk`Qn{`v_=&o(!U4bN zGtHV7J%S0f7LcICrJMGOa6Sl)^n6#p$&LgD#CRSbZmkFbVm@m2e)b%_*w^?RbS$(h zFrO|hYkA;$`TSD@nAhV^i^SuoR-M5+Io2)Fe;h&JBxIpk|3l~TW+Qu;eS#U*V&5+w zS~_~_8|EB@YWUn~UFit3EDE7Xmd$-uXU3%)L$~pO#~NGd8Rl~<`Fi7>TWVgRb9&G+!(sni{=YF-^YI$Yo#eK;icjIBO;N>Xje? zoObl}u-HUWi1^@-le$6+x1!+Kr_HmDZsKQma+Fu;Sh&ta?dcQj{7y8SgL-#4QX@1qv zXxqN^`;V9_7^XEt8+>0DqUM=ygBi3M>#QYv3ydJ%@~JlE9IYK9j{#Ok8kN}9R(3X- zkXeHA0CeMQ_K*=piD^`IV%E!3KMR;3$VTwQyFS*+YY%+jY*(4X!>HkZ`LO^(QWo{p z0Tpf{`617~nz+7jWva)*;|glSw5X{-r{|lKDg+PnGvR?94+qZ22xtz$%Ad=P&j%Zo z7D2$XpzG{Wm<-8{;uA}9&^N!XcmvVg`kdXPw%DYESK$Q%FeUd%>xGYMjU$e?q@5eq(bl=x97vj-K1@&+7TWY6(yU0*1^<5&79 zUZkQl0b?5M9*3A)rUJkJl5-{`r9k9#;qB#-cwt+E zTA}%9)@(Sk5R97`Ss2HUa{4HLfvf1%L0XRNSGU;-4h~R1Qj(IS7*g7kUHqkmSVvSS zm8X@!2!AKn@X|Xf=+X-u8QxZIApGMP!$EC^3#^gFvi@8dNoL@1bb1qd8En0YUwq0O zMuhyA{6TOHU}AWoK;K6FX3|=9-epBvOumiq$J>P8G)ilx2}REw^ilN)T%MuJgod|6 zC-I_|)~z9&@7?|K8AzYfmdzEif`fuk zFElXTMl<{aQRJE6QU;c918N8T6gegHW>{vug_YZfsX6Wkmd+-=_$ihf?=HSg>Bb%Uo#mcY)&3!cn zMdS0~3!EE;0pT7pXmo3$EF)V)4%u(ww2Kg>{@43$oRZ4ASJE zWL4@Abgb@K!c6(^2bPgb`gtp%0|ACWv17oGYmYT$e!@9n$>bGQ5_yP}Kq_bO59;el zOvtkOog8ZeI0S(~(UnCSW0j+}M@E`}r%NF6@tZI=R2B@EnhpZN4pd14;Gs4{TiZ1Y zHWxBk-@v83f%Z&kN>B3_NPlyc^AX$|=(IND0X{&3(_pf33d7Drtb`@bSR6+D>Sw)u zcI7WFDysII@Ky1U^!8zU+y@+rJRgC2$Y!=c5)1nm_fexhJ@CwZh7N9tx~-Rdh|5wh zN9wir+LWN!Zm8GgdA!_r>$&Ty(-6 zivQ>ixlvJ3stI%?75;fW-e=*h+ikzu57{sL%<*>Imcr1U0py%v1bK}{o=41Pe~v38 zgHo^hMlhVYoFA?I4Vw<`{R;>NrfU|%fo%Q%L9RZn^+E!0=k-GBH)#KZz`<*H1+g}- zw6hQ)PX2$f`HBX>%r_b&boc+g286-^*9Q!FNSOlcA4wYISAPz$E=7%`#NR?N-4Tb( z1`|}w8#y2hxZpl&T;JLl+&rAld>t6IX)fm_G!-!Uer-0cY-f zP&2KrQV;wskcQ^DSMoh6k;?`ILqSt#uQnf;@bII>sPH$x-d#OR zRm;8D+WqD78*iGM%nuAicE#`>)?qq*?}GZJrCG$#U~O3jf__iVe^@8lTsd^{1_!-R4H|eVVRbDX?b%cTf3s`jO ze)wwAk>a7)HUO-2U>$+R-$X=2CYzx+ETFHpe1oHf%F}k*H?k>kqn)(q$4AczoijEl zN&+EEh(d~J?iAA;uP;8r(#rTunKe{cTG$n;B3t$iPT{TWC>?UIDN#RoL zpeya<2nYM_Jyiy*LiEJXOWj0|=Clrfz5}HiPo4$_1n5znN69r^F;GyzaI8d?tFs}#)drQtulZ=fo*c5{OZomKjMeSJ@~Sj@b<9{D_ORt6qG-`1w4 zCR6L4ND@5uPoHI`3zhq4XRiUX(up#t(BgTY`6I^tuSsHlEx({y82V27_o)yuZR^3P z9DA&)`B*ry{4A;jUv$a4F;-VOjjot!HU}H}(|hdKIukGI{7$$!NDJnRR2ley^kZ;q z_&U&1-fNA$mZyfp1N6@R2CoJ@V=%L!?mle(y1KeXq$Z*RtniJC?sI!LyhWSz^z^xw z&{*7>8RH}OX1HNE-rnAzjX5XZ6$ng!w_Lmd3=lT^rv!RZNJlkG(Ij*C1lNvefzQ;CcOJS{FrX36qKLHWX@V9c4mQJqwmNrSlcsy zzj0oWm&W|j+QFe_qrs@T3H~jYkU=%k+iz-c5)RoI=cP5%M&?)1l2x9g15}ckjkGPB zaUbcM&K0NR+tVah5(6aTJD*Vf^@Me>z{YgoBaQF^>|Zd%D=YFsyT!K;>k<6wwV%EW zV5N$I)&5TV69CRO5YsGB58=@@biRr~ z%+@&^kBepuSDP3{Ejl3x)~*8Iz8^|ZXof`RGwqV&yv4C7J6b2qi?*g;N#;Q-0Y{0> zSe*gMtfuI6=I5V%NRBtIeFQe`GpeAQPzL7dT@5K0=v|BuW7-pvFslQ(e8Eftq<%)C+Tm=QVI} zygIF)0UrcfZJpLkYgN4-DQ?X|^o|>Blm8AhHz{KoiIV152?`M=P=jwi7V+HfS8;Vd zZ?+D)owsT!As8^COU)ptDroP*(62WXwLrl&S28NO%|x_MEFW>zwqamkxRBo;%L$1^ z`sb|zdq@xWRw84>woL&S?5N&$zE7Y`SO=jw1IYIX;U(MJ`5f_FfSSnxndW-rbq7^P zN5^1|(+xU0@)SmNncVPAVr;Dc`;JO9+Yze|3I-QcepBnJwX~*_S6^Pfneq0Y8n|l* zCVZjrwP6pFKh3~u!p6={K{B~B(X-B<;Za87lME*vw8G}$dd)XTC555emwTkQtfytv zQ^BeJXdmimeg}W)YB%SU3jFMxAsv`Jfer?F=KG`*E&2vj+v-R@sjuW@Z~kaI;*bC< zm>LrNG0%57k$gC5g!@kI6-`RVWX4@btesdomVRoc%}XjxHwR2I3?Q*?|JSq z!^*26C<9ao8Z*zRhzKg|VSfuKO~a&qcAdxx@UMza<90uIk>6YBOltc2`m2LYZkGWo z^lFuUGVOJqU|r`Yq7ZPM11pw9w++237B?weM^Xa_zP+U;?#4udm!JR-G-+T`411A) z9oS|CH*$wkxwTz3V*ww>o#qwE$U)dK1U4%L7$Y9bg}3=E$$?>nysn@wjKLVx{L(J0 zWO)O&X|n#PAz%dm*D(biusj9IxiAW^-%G3&T3^Ykmx5rMI!EBI9yOr@PEz3QDGCyk z=URR~CD{Z7ifX*VefMA(uNjw^XnIXht{7?f+m)ZLbgDq>KP?WJ`sid^Mb}lF7;c&wv7mikqDUhsmZzuVd9fnD zCcHBH4d?$3tIjlZuwuJw^Z&d6Fe4ApxL;rj^;kQtRouHGjUq)i@v*Uk%v=CjKNp2q zBvuxh1SNG3Y^}%dNmUi?R#2tM#C|l%(Vdw_p^k9*hI^cMk zPZq+s7;X~nB1>6cotupZ+<0P zEqPonicDq@OtI~Hu6B!T3A`VqB%YP12CeJE(%ia2aX$FCKq|n5+o_jxzUucR*L(#% z7vWq+JDF2TEFTVc-mqjM zxFNGyWPLtQ49K%}<14F)9bmjqg)$t^< zwR0dBXvf&ZRe)`v)kP%ZE1?N(t--Lr+_!d6V=e9(skqfglb_LSoV7cuxmiwiTE5%d0WGxDnTY|oE7WbLIC<&$Fc~zwT?_T?Noy*&yQhFhsyL8htGedTH zVS3vox<9b^KcQxjVn_s)b19&zPKzp2WucqHCWrWcg3)7e?}ec>b6QZCw18nLM1Yc; zs$X$Wyt!#3XKyI=qnNE$`o|l|--STq-DMrN@7i#JY>z%gSI4mSd??Qn8glDoX}(GR zOr)9qWB>D$AV(%Ge|%S|4k!5Trq;0duPf>!fGLo19s&o<@p7Y!ogGaZHRF`jr#0Zh z%m$~FyT;ClF}?f$v2U&5c<3Yx`JfY->@8bp+tqe|u#V`uVWr0~zU9u&-foX(kyB|1 z8YZ!twySI;C?@t6D_$NAORrm*d zQ|=b`1{=>}j!*18AH8dOXtaPXKw!=Jiy3nKho?eeDlyyTay zS`&1LGlP0MJv-%=x~a+64GnR&hs+;~W<>6{-V5uS3*RCyrxi}oQR^_`iOs53kmX40 zM*D^!G)nj{7B|ktWHNJfp(rb9V;2EiW(cNw(0-`5Na=tXkUa77K9z*h@ycFlL#b%Ut1R(|G@yYbCt3cK2B^n)5ZaUfjGqj_`mk?+ zSbZJnj&-U5;qW6O0_9$@8I+Qicd1fBw`%(R%B|8Q&12Ruen(n`phC_W=n!5pXbngJ zzo+^77v2c*8!j$fEkZ0Taafs3q*hxP0RUKBJZd%odt z0V+6e%ZaRdgl0v~SBzCvm$5CT$LrJV6|^qXIY`&5rR{(r?Rr zqbG@@gIuhfzunPsS1zuz4xV1Dn&W!0)-LkKg>^5E?)pBTyK#QM z#x`>J`)hJ)gG^wU!ojaXuFHIO7Uj;zKA3qHTq=LbcP;XC6a_)5@AH0BThHK>^nUrX zr?cUCD=vA1*ST?YWK};;RDD%DZ|O_GZZfh0IGoo_VEG3i;jI-NPL<4qwfqRCMo3I> zet!Noocr(!gu@l?Gcyl?CswTP_2PZ!`t6D@in<4twGD{wC$W(dEuUSTtm|fG+sw)) zGW+j>&mS|a1#Eu?V`%BDhRTRrtk>_m#fLmsuxrgGsVTX0n~&RHIET!Cw~~8$Hd3l5 z3qIW!dEM%@2;X;2!2Sl<43J#sb4`n_J;wL25)>Tp3?r~Xpf=Ua!`85QS3hSKZ6VI1bcSm1d zwmfQQacQAMd~)AAhV`Q5G*C6yI+Vrfwv|?d>8>pbS+Ce{jK=8lNOoBet7`KM2)QiU zxg97Pl0^#MJi^`47X6;Jut1>hzr@6vBAn%Sx1JtZ zenm(NPrOwY8dK<2c48jaJY`MS;9pejzq>7l>6jJlA-8Er_4pLjZY2N3;3=V*p~9{* zTLBbho8wLGpK80aebWn4Q3@N^3vWF;CN{jN2dGUyK(+#J>Hq<%COs2I+F2G~%ZYrn z^f|lYaaRu#a+}UI|AUEwm@e2Y)?j3fA;+a;Gu(LFoG!Ezd*1Z8=ztX+k-QE5Dl<4S8 zU?n@@1mFneJ+Y`0Y*KIEckXpb8&CfzTvF;|(iU$E=UVKF_~tx{N@zh6C;evoJU+-~w%+4pnpjix_;J`ZlelowU{P=FlAwr$6X z*b(LOI74&zkrH84;RQioOS+V{8c9dRG}%B8Z)Qs4*Q$uV@FLWh?0>=tg%nUZWFw5~ zGh`bzWLW`vinJsof>^n);h_=fKGur7<}n@1hEU@mp=MH5H=RrOEGGAZm3TG62dd|; zUx2y_4F}eLb;JlW`R70p`%9I3q4+49bwshFr!iB?2u>p5>>Zy#V&D_%P^N2G+*d-W zCm9f_4$y$h8)pw=h_QRDw0c8==Eip*a@^~EK3;Ix{;4FG!ZQK$tp9FCf0q2_H|03? z0Xg(x*!!mLiJ=O^-HFV~^!OKdn9Fwk0nJP&iKaN5udqU7gdC3=SzoiQcScm{LwO*T zG14Z=Ibn%&Y$mT!w?+NnRd`0bTVWj)iNI@|fZLt7h>(Lou( z{=-7Tuk-CSgLCYaJ)iDNfy@edL857-;=oHMH<=&0zVXrq1ZCr@7=C;deCBuA1mp4D zHNnROb8NJ>4D{d^E;|YuUMhwsJ4otmoeuWKafTub*0aaD z<8Hl7d5R!+?Jcs7q3WbJXG2?2{9$KIH%qyMiP|imt5zHaS*5pXVs!^ZA0yx9-4(T$ zs!DumoG%spA!fV!i44Xr~I&stlc%)pQ%7KoZ&||-oOvl+aCG$cHjf$^?ubb)ad?f z70ya9CQW{bNU~@ghg20Rsb!A_8%&D7|9$K!{1BQh97&k@3!MP5v=B^@eMt;sC5`@z-PVIivsNrPr>wTY}udTJpP{5If3#+p15^Nw}rA!W;A{|A(zMHa9 zwUoB94(F$2Qrkwe`1^E@N$zZqyrdJrUd-xJ`{SUb8pL(bkE& z65R1gCC}84kyu?QEL%kTXOs*#3hy~;5v*>j!EU<>L7Nk>TNYD-uUo{LeVBglO&gaZ z28xC)tSF?N?@dRDLw%h(Hhm@ddWhnr^Q9c)hlQn9DB^0Agb#X!g4H;2>6^Cw6`#s8 z$G3f3QkuW7nO2jNm}uaO+eEw<#3~xFDBQH_nn`xg%%oS~G*p+3k52QrWJ0Yo zJW=T?YA=|a(JjVyVb9MNb1d~_5k|yi;`5KArRi5U4-3;DjY?&8JsaE!OBc6#>geXpkL-Ctjl*_wSNySp}fCK2f(HrVHg&fk_XME9nMjQ}x%S;S>o zy00p>@10UNO2I-_0FUeN8H~#F2zgG^0G&DVnT(;h7A6?~zPQTiY_7|q0ET?t;lg5q zzMAkpdZM2$CQbe?iQEf6kw(q^fsHA~kX13t@ltUeq-J^xkY0#xBeEG#f(p7FMto*r zAQM#_+6SBrs% zp6ZFR;qMUKiEn2Q`G(2V``l7rsn)w0N}6$L-ya&+aG@ho=bSph?<{$sonY0k`KbAJ z8&`kj{A-&I@>gospe)U;j0J)0Hi>Un8|8D+{s|xY+o+!FypEkxbqIG)BY0{qRG#}b z-1fDC=(1t;7j-V{NGZ*1*);#gw8QL}v442WhefH?jNa-3;wV zbNoTVm==$xbrF(^&~ z0!i1D*Wc(DsgGA`AVxt#Oh8fq3!%bUPThdo80;++Un>Ix1C54p!!yS;CO?g#vq@(b zpN+a%e~4QxE$A&Wq=1TO$AzR_la~YwE)PmyRU$e~?f52O=f%K0CTo7c(V0XWd=*#O zn)d>?&#M{^h2g`*&kXC#WEfNj)MDQcb4IeO&_xXh#wtVjtaX%n3Y99wejQpuk;$dV z9+t#nPa3+8^8L~=IWO>gtjl&(Ri_19byfUVhSZWJ%7fBMeBwk1wCdS%v;$FvrbQ2A zg^lqQ=J$>^ZH#o$lP_^A5mbH;e!$@`d@O+N{UXP=^z91~&9zP#9OGIv63PAzfhp|+ zX~z6meuuT?YtP2}ep!#5!~WMJsLz;xzI~WC^%HU5Na`xC=rCQ(acSMSeqJ5$_R+Pi z+uN@=yLvtTyf`(*(??vyN4-_viC|!bDI+6N)NK5aRJg%k_nBeo&c{pNf`);Rq5O(e z0z5yLOrSZnz`NFR(M<(;kz?U$Bb*%C@6(y{rI;&iPt$1-c)EG`?JW=GL@cBnP0>C} zd-Qg7^s|#|I|^FF;br~^us+jy0ylF0_}z(6Kv7u@n-D`LNJj|zS#ZL)^7k5|p+!3M z&k+ra>1q)3Kj+tUjWL(x1Q{Y{6gCGGj)bH7(`7L}$zfARbA&RHYx)dW6ihRYDHAsz-@ zhIm1ifE3@klq5FhQk3t4&T;W}+Ty(RIpdGc#P-^c2R9L+FQ2ZJh(i8%ex6O5bA|8GonOnx)?}3k<{&4amp1PGG%PsPhI$R zI+pzWoyfOF%f`%;uLG?B@8^qnsS`utVznLyhjR78_-(4|)mbm$4O(v(&+bJ_=eWgMnA>MG>^ZGZFl5o_$RKGDp>ElQ8iD%EbmXPb<<9O%0mMzQOKn z%FQ_O(csFs{~m?zHZvKz3|91ZhHq#J;mEF8!ggSo0Bn z*pmvi7%>cI0}Lc_zJl$XF{idnL!Og^M6~f66dLYfai6$s+N}J$u=Jeu3Oo!2;^Ot) z7<_)?+w#ceu(VBlp5v$=Att3GE9k>|;qk~E77LLH!jh5+sR&^?{VIk*>x5Tp=W7R3<6)|+4}I>2$r??N%&AeGFDCK~$c$J$_UK10%ETUv5CQaW>si8Ab> zoCQrr?Cnkx$@46gfrfQRh+JO%{0KC@aokuBp22CZV>Hy_Q_Rqid;+)pQ##JTU6=%C zAp?1|oJKEjVwQMr38U92wJ6~tyN}^U}VF{0s&Oj>^!3Lf8(TbryxHk^B60cbqhL`hjQYpYzcp1;Wri{KgK(wJRWn zToC>5Srgh&Mj3lYOFx*w1>)eX(-&zL7cDI>?$Xv zQ!{KTx%>9I4jkpq6#f<5JQ8wa46E>Om?aNtwnEW=$0GpcAk&Jw%>{dC|L@(vjj!Jc zJWXA`H9r5J-#N$#Jk{)OO_lx6uq23E;6FSA$HgCwp2fCYH$$(Ny@BS6Ey-f^BE~D3 z_N!sg-2Z+(5tD8hPXZS2)2A@@1Okr13(#H2JF&MuD)1AW-b=!PEXKMIG8gquJmyF9 zS(>RzPC}4}zm{r64|iYWT#o}omG^*JZ%)0v-f+45rr=@a3VAW|#NfGMDo5#+z=#Bp zp(0vNJj%tw@KaeOA+N4A9iB4daiBvE@(c(80uuazn@N832^LP@z~M+>uBM-eXhPm{ zk2n;L+E!@9b=_H#I0T^cuWh;u>gIl!K(XnO$w2`{4fYPdAX47ufP+~<>C)s0Ea-X2KTDp%jSohJzSJkZ$5%6rFF~kU&lk- zNBg5$;&YkIgkYJ;fvE%)P!T^up;M9`gG41S5W0;m7qHg?ks$j79+y*ThNkzD!`owT z(%o%AB-Uwv#wLWWHYn7*@B|dK2lTZ)BqIf=yFhe--|pAIu1KXpZ}e80n;viFlKa^p^fPnoo;0^Zz*v3;HC=LGJ$9wQT|axoQPC-V|K}9zq_hFrW*Z{ZjP|8anjDo0q z(5$=`d;1w$+(DAn!|#WhQ9B^>H2#?Ge%d={T`C7G*9aVC#tVyX%`)g+6mRf!U!wyc z8ma6U*&D!509k|sh*%I}NTx9tHzU#zKXApC6pJ@pZ@B+(sG5CR!gTUlGvLvOMt zmG%H(7LbjHmEO~QCBTL6FrfhS4jw@H5zhZ`1;QtgEzI zx4#(_7!Rj;jEZ6jo`$jv1)<;ZgDNtdTPfo}PE|F;6~f+8)euEt?C41+Ko&^gh#mXJ z9*@}oZZ9V?z;Frp_@j-*J{DZ3J){8g^XJ}zpkPN1h$a%e0b?Lmpwn)~*RLhVn*-X_ zt|KJ*-USlS!;6u+*^v)~Gd>>x(IK{lHe@{)N&U3_->so-6zZ68&`eq}#cu;D@7 zL4iQj8<55-7LG5OJQI1#eNg=}ha!6n5-=MfX(Sz~7oGuYZ%u5D^B%&Rf@Q^f`7Hl< zJ)Aq-av&&|?1>i840yy)yn`bIGUlW3kC{L94;=p^736qcDI9>deNtFdk7xx zV11EACVa**dB|IlhC72GciZKmIO@l_jzNxG_8>D>C|E z{yR#1rBeF|i60Y4g|O}2i*0!u9i$iH6!cv(C^JG>j+$ml%gutCG7~ujiy=3xFr1c^ z`t%WeAO!X!frX2@0Sb1eIZTA zzW{5P4zV+#>SJ9WCGPoZIJv~G&ah_-Ffwh;y%fpvbAqk*t#seiA%h z0m6a8rZ7=b#?(wn99CMR`Jm|9F%Y{TtNKo4ErPs~C0~$VgXYI96oW|rn@mDiYx^p) zBG)dPf)oYMs*#DUK|!vNbV|)Ql@j(4+9a^r#$aHFD6pH*?Gs2&<9ocb`)a=}Q~Nv7 zIBiviOzP`ROpmY#ZDl8(p9rEKuBbo4n|S;w^xp%sbU$!$$*xGHFEpXQ8~~e&i}cO~ z9%p+bW>WMkb>)U%Er>ZQmfnM$SS4GJ^)QZK8n-~J4M(*eKo&gmXOXwa3Ov1^R+|Ox zmjuC6zeONn4rP43(@(n8V;HUQ>x*gZMsyaN{FAV+*^aMfZ!YGoNE_vYM<+vJT^4{6 z0MyzD_N8itQ%y|8XGiH1SdX?v{yCB_M;!_TL&$4w{5U4NpLgVzl3MezR|QcqIH6H6 z1hPmA0>!*uvh4EnF>Ef%pvmrwUZgxwDg83~Ooe*$6*+JSm}L4qcq;B(SF(J?MjLp4 z0yCD^=_jx>qK38`Q9q2u6vkx~*u;|8-K4$cTRRu=*`g2d(G{h0Jj<< z7Y+~0oh(!jH#wlO86HG38<#TjHc!Gb%{Ljt=cVJt+zE>0VR2p47rk=e*GdqRcz6CJ zLWX$R7`QEuGY|S4~eeeU;X;*+@2#t#IdM~mPH^K1WlJs0` zsKehD4SCW=C_LIcG+&Y0RSk;QWeb|D>ncRjBMWv~Bu%z4n1rlJZb<{oRHwtIO5gTOcMuObQxf^FQV zU%mDiUpunIeIY3MLYXaB_g+D+bPXQ`XtYVYLq8+0aT}Py6DP4=3EI8HQ`;bZ?i*DP z7cs!f@ETDJRxeNQE#-s`;!%o8Hha&bMDRY93mSZ(6n%xurP}udWfNV3q&BAhd<_NwjP%xVLfj|~C>}>r`V4bS z=%fVuw-I4V7s@d#5&q3Pxyf&laC*L83L5zKSTYnzFdlx{(C8K zC>b%p9Zv>H{s)TqO9MMs=XTTwVI}_F1sy=D|G%+ie5PM$a@zc#Q{jQn_fo(v=s%-~ z6j()<7r;7PYOwwHQh%NlV+1~fQ{=yBGs7A<$S<6ZH2(*22|xjt6?z08g46yESiTO- wgnz?jsm6a#C5AvxC=+A|0{c5)FiVfm&BZv431Dg*pMXD-qOu~TLOMSG4+$y0LI3~& literal 0 HcmV?d00001 diff --git a/cachelib/allocator/BackgroundEvictor-inl.h b/cachelib/allocator/BackgroundEvictor-inl.h new file mode 100644 index 0000000000..9cec5d3930 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor-inl.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace facebook { +namespace cachelib { + + +template +BackgroundEvictor::BackgroundEvictor(Cache& cache, + std::shared_ptr strategy) + : cache_(cache), + strategy_(strategy) +{ +} + +template +BackgroundEvictor::~BackgroundEvictor() { stop(std::chrono::seconds(0)); } + +template +void BackgroundEvictor::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundEvictor interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundEvictor::setAssignedMemory(std::vector> &&assignedMemory) +{ + XLOG(INFO, "Class assigned to background worker:"); + for (auto [tid, pid, cid] : assignedMemory) { + XLOGF(INFO, "Tid: {}, Pid: {}, Cid: {}", tid, pid, cid); + } + + mutex.lock_combine([this, &assignedMemory]{ + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundEvictor::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this]{ + return assignedMemory_; + }); + + unsigned int evictions = 0; + std::set classes{}; + auto batches = strategy_->calculateBatchSizes(cache_,assignedMemory); + + for (size_t i = 0; i < batches.size(); i++) { + const auto [tid, pid, cid] = assignedMemory[i]; + const auto batch = batches[i]; + + classes.insert(cid); + const auto& mpStats = cache_.getPoolByTid(pid,tid).getStats(); + + if (!batch) { + continue; + } + + stats.evictionSize.add(batch * mpStats.acStats.at(cid).allocSize); + + //try evicting BATCH items from the class in order to reach free target + auto evicted = + BackgroundEvictorAPIWrapper::traverseAndEvictItems(cache_, + tid,pid,cid,batch); + evictions += evicted; + evictions_per_class_[tid][pid][cid] += evicted; + } + + stats.numTraversals.inc(); + stats.numEvictedItems.add(evictions); + stats.totalClasses.add(classes.size()); +} + +template +BackgroundEvictionStats BackgroundEvictor::getStats() const noexcept { + BackgroundEvictionStats evicStats; + evicStats.numEvictedItems = stats.numEvictedItems.get(); + evicStats.runCount = stats.numTraversals.get(); + evicStats.evictionSize = stats.evictionSize.get(); + evicStats.totalClasses = stats.totalClasses.get(); + + return evicStats; +} + +template +std::map>> +BackgroundEvictor::getClassStats() const noexcept { + return evictions_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundEvictor.h b/cachelib/allocator/BackgroundEvictor.h new file mode 100644 index 0000000000..7583732127 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/PeriodicWorker.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" +#include "cachelib/common/AtomicCounter.h" + + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the eviction. +template +struct BackgroundEvictorAPIWrapper { + + static size_t traverseAndEvictItems(C& cache, + unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + return cache.traverseAndEvictItems(tid,pid,cid,batch); + } +}; + +struct BackgroundEvictorStats { + // items evicted + AtomicCounter numEvictedItems{0}; + + // traversals + AtomicCounter numTraversals{0}; + + // total class size + AtomicCounter totalClasses{0}; + + // item eviction size + AtomicCounter evictionSize{0}; +}; + +// Periodic worker that evicts items from tiers in batches +// The primary aim is to reduce insertion times for new items in the +// cache +template +class BackgroundEvictor : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param target_free the target amount of memory to keep free in + // this tier + // @param tier id memory tier to perform eviction on + BackgroundEvictor(Cache& cache, + std::shared_ptr strategy); + + ~BackgroundEvictor() override; + + BackgroundEvictionStats getStats() const noexcept; + std::map>> getClassStats() const noexcept; + + void setAssignedMemory(std::vector> &&assignedMemory); + + private: + std::map>> evictions_per_class_; + + // cache allocator's interface for evicting + + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + BackgroundEvictorStats stats; + + std::vector> assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundEvictor-inl.h" diff --git a/cachelib/allocator/BackgroundEvictorStrategy.h b/cachelib/allocator/BackgroundEvictorStrategy.h new file mode 100644 index 0000000000..1d05a801bb --- /dev/null +++ b/cachelib/allocator/BackgroundEvictorStrategy.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Base class for background eviction strategy. +class BackgroundEvictorStrategy { + +public: + virtual std::vector calculateBatchSizes(const CacheBase& cache, + std::vector> acVec) = 0; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundPromoter-inl.h b/cachelib/allocator/BackgroundPromoter-inl.h new file mode 100644 index 0000000000..daa6ae0a93 --- /dev/null +++ b/cachelib/allocator/BackgroundPromoter-inl.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace facebook { +namespace cachelib { + + +template +BackgroundPromoter::BackgroundPromoter(Cache& cache, + std::shared_ptr strategy) + : cache_(cache), + strategy_(strategy) +{ +} + +template +BackgroundPromoter::~BackgroundPromoter() { stop(std::chrono::seconds(0)); } + +template +void BackgroundPromoter::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundPromoter interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundPromoter::setAssignedMemory(std::vector> &&assignedMemory) +{ + XLOG(INFO, "Class assigned to background worker:"); + for (auto [tid, pid, cid] : assignedMemory) { + XLOGF(INFO, "Tid: {}, Pid: {}, Cid: {}", tid, pid, cid); + } + + mutex.lock_combine([this, &assignedMemory]{ + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundPromoter::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this]{ + return assignedMemory_; + }); + + unsigned int promotions = 0; + std::set classes{}; + + auto batches = strategy_->calculateBatchSizes(cache_,assignedMemory); + + for (size_t i = 0; i < batches.size(); i++) { + const auto [tid, pid, cid] = assignedMemory[i]; + const auto batch = batches[i]; + + + classes.insert(cid); + const auto& mpStats = cache_.getPoolByTid(pid,tid).getStats(); + if (!batch) { + continue; + } + + // stats.promotionsize.add(batch * mpStats.acStats.at(cid).allocSize); + + //try evicting BATCH items from the class in order to reach free target + auto promoted = + BackgroundPromoterAPIWrapper::traverseAndPromoteItems(cache_, + tid,pid,cid,batch); + promotions += promoted; + promotions_per_class_[tid][pid][cid] += promoted; + } + + stats.numTraversals.inc(); + stats.numPromotedItems.add(promotions); + // stats.totalClasses.add(classes.size()); +} + +template +BackgroundPromotionStats BackgroundPromoter::getStats() const noexcept { + BackgroundPromotionStats promoStats; + promoStats.numPromotedItems = stats.numPromotedItems.get(); + promoStats.runCount = stats.numTraversals.get(); + + return promoStats; +} + +template +std::map>> +BackgroundPromoter::getClassStats() const noexcept { + return promotions_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundPromoter.h b/cachelib/allocator/BackgroundPromoter.h new file mode 100644 index 0000000000..04e0e7d187 --- /dev/null +++ b/cachelib/allocator/BackgroundPromoter.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/PeriodicWorker.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" +#include "cachelib/common/AtomicCounter.h" + + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the promotion. +template +struct BackgroundPromoterAPIWrapper { + + static size_t traverseAndPromoteItems(C& cache, + unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + return cache.traverseAndPromoteItems(tid,pid,cid,batch); + } +}; + +struct BackgroundPromoterStats { + // items evicted + AtomicCounter numPromotedItems{0}; + + // traversals + AtomicCounter numTraversals{0}; + + // total class size + AtomicCounter totalClasses{0}; + + // item eviction size + AtomicCounter promotionSize{0}; +}; + +template +class BackgroundPromoter : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param target_free the target amount of memory to keep free in + // this tier + // @param tier id memory tier to perform promotin from + BackgroundPromoter(Cache& cache, + std::shared_ptr strategy); + // TODO: use separate strategy for eviction and promotion + + ~BackgroundPromoter() override; + + // TODO + BackgroundPromotionStats getStats() const noexcept; + std::map>> getClassStats() const noexcept; + + void setAssignedMemory(std::vector> &&assignedMemory); + + private: + std::map>> promotions_per_class_; + + // cache allocator's interface for evicting + + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + BackgroundPromoterStats stats; + + std::vector> assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundPromoter-inl.h" diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index b00302086b..8dc0166ecf 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -35,6 +35,7 @@ add_library (cachelib_allocator CCacheManager.cpp ContainerTypes.cpp FreeMemStrategy.cpp + FreeThresholdStrategy.cpp HitsPerSlabStrategy.cpp LruTailAgeStrategy.cpp MarginalHitsOptimizeStrategy.cpp diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index ac985a7ae2..f021eb0aaa 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -93,6 +93,12 @@ class CacheBase { // // @param poolId The pool id to query virtual const MemoryPool& getPool(PoolId poolId) const = 0; + + // Get the reference to a memory pool using a tier id, for stats purposes + // + // @param poolId The pool id to query + // @param tierId The tier of the pool id + virtual const MemoryPool& getPoolByTid(PoolId poolId, TierId tid) const = 0; // Get Pool specific stats (regular pools). This includes stats from the // Memory Pool and also the cache. diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 8e8583b4a8..2a10ee30f6 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -340,6 +340,18 @@ void CacheAllocator::initWorkers() { config_.poolOptimizeStrategy, config_.ccacheOptimizeStepSizePercent); } + + if (config_.backgroundEvictorEnabled()) { + startNewBackgroundEvictor(config_.backgroundEvictorInterval, + config_.backgroundEvictorStrategy, + config_.backgroundEvictorThreads); + } + + if (config_.backgroundPromoterEnabled()) { + startNewBackgroundPromoter(config_.backgroundPromoterInterval, + config_.backgroundPromoterStrategy, + config_.backgroundPromoterThreads); + } } template @@ -362,7 +374,24 @@ CacheAllocator::allocate(PoolId poolId, creationTime = util::getCurrentTimeSec(); } return allocateInternal(poolId, key, size, creationTime, - ttlSecs == 0 ? 0 : creationTime + ttlSecs); + ttlSecs == 0 ? 0 : creationTime + ttlSecs, false); +} + +template +bool CacheAllocator::shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid) +{ + // TODO: should we also work on lower tiers? should we have separate set of params? + if (tid == 1) return false; + return getAllocationClassStats(tid, pid, cid).approxFreePercent <= config_.lowEvictionAcWatermark; +} + +template +size_t CacheAllocator::backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers) +{ + XDCHECK(numWorkers); + + // TODO: came up with some better sharding (use some hashing) + return (tid + pid + cid) % numWorkers; } template @@ -372,7 +401,8 @@ CacheAllocator::allocateInternalTier(TierId tid, typename Item::Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime) { + uint32_t expiryTime, + bool fromEvictorThread) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -388,13 +418,18 @@ CacheAllocator::allocateInternalTier(TierId tid, (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); + + if (backgroundEvictor_.size() && !fromEvictorThread && (memory == nullptr || shouldWakeupBgEvictor(tid, pid, cid))) { + backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp(); + } + // TODO: Today disableEviction means do not evict from memory (DRAM). // Should we support eviction between memory tiers (e.g. from DRAM to PMEM)? if (memory == nullptr && !config_.disableEviction) { memory = findEviction(tid, pid, cid); } - ItemHandle handle; + WriteHandle handle; if (memory != nullptr) { // At this point, we have a valid memory allocation that is ready for use. // Ensure that when we abort from here under any circumstances, we free up @@ -431,18 +466,71 @@ CacheAllocator::allocateInternalTier(TierId tid, } template -typename CacheAllocator::WriteHandle -CacheAllocator::allocateInternal(PoolId pid, +TierId +CacheAllocator::getTargetTierForItem(PoolId pid, typename Item::Key key, uint32_t size, uint32_t creationTime, uint32_t expiryTime) { - auto tid = 0; /* TODO: consult admission policy */ - for(TierId tid = 0; tid < numTiers_; ++tid) { - auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); - if (handle) return handle; + if (numTiers_ == 1) + return 0; + + if (config_.forceAllocationTier != UINT64_MAX) { + return config_.forceAllocationTier; } - return {}; + + const TierId defaultTargetTier = 0; + + const auto requiredSize = Item::getRequiredSize(key, size); + const auto cid = allocator_[defaultTargetTier]->getAllocationClassId(pid, requiredSize); + + auto freePercentage = getAllocationClassStats(defaultTargetTier, pid, cid).approxFreePercent; + + // TODO: COULD we implement BG worker which would move slabs around + // so that there is similar amount of free space in each pool/ac. + // Should this be responsibility of BG evictor? + + if (freePercentage >= config_.maxAcAllocationWatermark) + return defaultTargetTier; + + if (freePercentage <= config_.minAcAllocationWatermark) + return defaultTargetTier + 1; + + // TODO: we can even think about creating different allocation classes for PMEM + // and we could look at possible fragmentation when deciding where to put the item + if (config_.sizeThresholdPolicy) + return requiredSize < config_.sizeThresholdPolicy ? defaultTargetTier : defaultTargetTier + 1; + + // TODO: (e.g. always put chained items to PMEM) + // if (chainedItemsPolicy) + // return item.isChainedItem() ? defaultTargetTier + 1 : defaultTargetTier; + + // TODO: + // if (expiryTimePolicy) + // return (expiryTime - creationTime) < expiryTimePolicy ? defaultTargetTier : defaultTargetTier + 1; + + // TODO: + // if (keyPolicy) // this can be based on key length or some other properties + // return getTargetTierForKey(key); + + // TODO: + // if (compressabilityPolicy) // if compresses well store in PMEM? latency will be higher anyway + // return TODO; + + // TODO: only works for 2 tiers + return (folly::Random::rand32() % 100) < config_.defaultTierChancePercentage ? defaultTargetTier : defaultTargetTier + 1; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::allocateInternal(PoolId pid, + typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime, + bool fromEvictorThread) { + auto tid = getTargetTierForItem(pid, key, size, creationTime, expiryTime); + return allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromEvictorThread); } template @@ -1401,6 +1489,62 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, return true; } +template +bool CacheAllocator::moveRegularItemForPromotion(Item& oldItem, + ItemHandle& newItemHdl) { + util::LatencyTracker tracker{stats_.moveRegularLatency_}; + + if (!oldItem.isAccessible() || oldItem.isExpired()) { + return false; + } + + XDCHECK_EQ(newItemHdl->getSize(), oldItem.getSize()); + + if (config_.moveCb) { + // Execute the move callback. We cannot make any guarantees about the + // consistency of the old item beyond this point, because the callback can + // do more than a simple memcpy() e.g. update external references. If there + // are any remaining handles to the old item, it is the caller's + // responsibility to invalidate them. The move can only fail after this + // statement if the old item has been removed or replaced, in which case it + // should be fine for it to be left in an inconsistent state. + config_.moveCb(oldItem, *newItemHdl, nullptr); + } else { + std::memcpy(newItemHdl->getWritableMemory(), oldItem.getMemory(), + oldItem.getSize()); + } + + auto predicate = [this](const Item& item) { + // if inclusive cache is allowed, replace even if there are active users. + return config_.numDuplicateElements > 0 || item.getRefCount() == 0; + }; + if (!accessContainer_->replaceIf(oldItem, *newItemHdl, predicate)) { + return false; + } + + // Inside the MM container's lock, this checks if the old item exists to + // make sure that no other thread removed it, and only then replaces it. + if (!replaceInMMContainer(oldItem, *newItemHdl)) { + accessContainer_->remove(*newItemHdl); + return false; + } + + // Replacing into the MM container was successful, but someone could have + // called insertOrReplace() or remove() before or after the + // replaceInMMContainer() operation, which would invalidate newItemHdl. + if (!newItemHdl->isAccessible()) { + removeFromMMContainer(*newItemHdl); + return false; + } + + // no one can add or remove chained items at this point + if (oldItem.hasChainedItem()) { + throw std::runtime_error("Not supported"); + } + newItemHdl.unmarkNascent(); + return true; +} + template bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, ItemHandle& newItemHdl) { @@ -1608,21 +1752,35 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( return true; } +template +bool CacheAllocator::shouldEvictToNextMemoryTier( + TierId sourceTierId, TierId targetTierId, PoolId pid, Item& item) +{ + if (config_.disableEvictionToMemory) + return false; + + // TODO: implement more advanced admission policies for memory tiers + return true; +} + template typename CacheAllocator::WriteHandle CacheAllocator::tryEvictToNextMemoryTier( - TierId tid, PoolId pid, Item& item) { - if(item.isChainedItem()) return {}; // TODO: We do not support ChainedItem yet + TierId tid, PoolId pid, Item& item, bool fromEvictorThread) { if(item.isExpired()) return acquire(&item); - TierId nextTier = tid; // TODO - calculate this based on some admission policy + TierId nextTier = tid; while (++nextTier < numTiers_) { // try to evict down to the next memory tiers + if (!shouldEvictToNextMemoryTier(tid, nextTier, pid, item)) + continue; + // allocateInternal might trigger another eviction auto newItemHdl = allocateInternalTier(nextTier, pid, item.getKey(), item.getSize(), item.getCreationTime(), - item.getExpiryTime()); + item.getExpiryTime(), + fromEvictorThread); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); @@ -1634,12 +1792,48 @@ CacheAllocator::tryEvictToNextMemoryTier( return {}; } +template +bool +CacheAllocator::tryPromoteToNextMemoryTier( + TierId tid, PoolId pid, Item& item, bool fromEvictorThread) { + TierId nextTier = tid; + while (nextTier > 0) { // try to evict down to the next memory tiers + auto toPromoteTier = nextTier - 1; + --nextTier; + + // allocateInternal might trigger another eviction + auto newItemHdl = allocateInternalTier(toPromoteTier, pid, + item.getKey(), + item.getSize(), + item.getCreationTime(), + item.getExpiryTime(), + fromEvictorThread); + + if (newItemHdl) { + XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); + if (moveRegularItemForPromotion(item, newItemHdl)) { + return true; + } + } + } + + return false; +} + template typename CacheAllocator::WriteHandle -CacheAllocator::tryEvictToNextMemoryTier(Item& item) { +CacheAllocator::tryEvictToNextMemoryTier(Item& item, bool fromEvictorThread) { + auto tid = getTierId(item); + auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; + return tryEvictToNextMemoryTier(tid, pid, item, fromEvictorThread); +} + +template +bool +CacheAllocator::tryPromoteToNextMemoryTier(Item& item, bool fromBgThread) { auto tid = getTierId(item); auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; - return tryEvictToNextMemoryTier(tid, pid, item); + return tryPromoteToNextMemoryTier(tid, pid, item, fromBgThread); } template @@ -2297,6 +2491,16 @@ PoolId CacheAllocator::addPool( setRebalanceStrategy(pid, std::move(rebalanceStrategy)); setResizeStrategy(pid, std::move(resizeStrategy)); + if (backgroundEvictor_.size()) { + for (size_t id = 0; id < backgroundEvictor_.size(); id++) + backgroundEvictor_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundEvictor_.size(), 0)); + } + + if (backgroundPromoter_.size()) { + for (size_t id = 0; id < backgroundPromoter_.size(); id++) + backgroundPromoter_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundPromoter_.size(), 1)); + } + return pid; } @@ -2361,6 +2565,10 @@ void CacheAllocator::createMMContainers(const PoolId pid, .getAllocsPerSlab() : 0); for (TierId tid = 0; tid < numTiers_; tid++) { + if constexpr (std::is_same_v || std::is_same_v) { + config.lruInsertionPointSpec = config_.memoryTierConfigs[tid].lruInsertionPointSpec ; + config.markUsefulChance = config_.memoryTierConfigs[tid].markUsefulChance; + } mmContainers_[tid][pid][cid].reset(new MMContainer(config, compressor_)); } } @@ -2415,7 +2623,7 @@ std::set CacheAllocator::getRegularPoolIds() const { folly::SharedMutex::ReadHolder r(poolsResizeAndRebalanceLock_); // TODO - get rid of the duplication - right now, each tier // holds pool objects with mostly the same info - return filterCompactCachePools(allocator_[0]->getPoolIds()); + return filterCompactCachePools(allocator_[currentTier()]->getPoolIds()); } template @@ -2828,7 +3036,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { oldItem.getKey(), oldItem.getSize(), oldItem.getCreationTime(), - oldItem.getExpiryTime()); + oldItem.getExpiryTime(), + false); if (!newItemHdl) { return {}; } @@ -2961,14 +3170,14 @@ void CacheAllocator::evictForSlabRelease( template typename CacheAllocator::ItemHandle CacheAllocator::evictNormalItem(Item& item, - bool skipIfTokenInvalid) { + bool skipIfTokenInvalid, bool fromEvictorThread) { XDCHECK(item.isMoving()); if (item.isOnlyMoving()) { return ItemHandle{}; } - auto evictHandle = tryEvictToNextMemoryTier(item); + auto evictHandle = tryEvictToNextMemoryTier(item, fromEvictorThread); if(evictHandle) return evictHandle; auto predicate = [](const Item& it) { return it.getRefCount() == 0; }; @@ -3353,6 +3562,8 @@ bool CacheAllocator::stopWorkers(std::chrono::seconds timeout) { success &= stopPoolResizer(timeout); success &= stopMemMonitor(timeout); success &= stopReaper(timeout); + success &= stopBackgroundEvictor(timeout); + success &= stopBackgroundPromoter(timeout); return success; } @@ -3633,6 +3844,8 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.nvmUpTime = currTime - getNVMCacheCreationTime(); ret.reaperStats = getReaperStats(); + ret.evictionStats = getBackgroundEvictorStats(); + ret.promotionStats = getBackgroundPromoterStats(); ret.numActiveHandles = getNumActiveHandles(); return ret; @@ -3736,6 +3949,7 @@ bool CacheAllocator::startNewPoolRebalancer( freeAllocThreshold); } + template bool CacheAllocator::startNewPoolResizer( std::chrono::milliseconds interval, @@ -3773,6 +3987,64 @@ bool CacheAllocator::startNewReaper( return startNewWorker("Reaper", reaper_, interval, reaperThrottleConfig); } +template +auto CacheAllocator::getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid) +{ + std::vector> asssignedMemory; + // TODO: for now, only evict from tier 0 + auto pools = filterCompactCachePools(allocator_[tid]->getPoolIds()); + for (const auto pid : pools) { + const auto& mpStats = getPoolByTid(pid,tid).getStats(); + for (const auto cid : mpStats.classIds) { + if (backgroundWorkerId(tid, pid, cid, numWorkers) == evictorId) { + asssignedMemory.emplace_back(tid, pid, cid); + } + } + } + return asssignedMemory; +} + +template +bool CacheAllocator::startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + backgroundEvictor_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], interval, strategy); + result = result && ret; + + if (result) { + backgroundEvictor_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundEvictor_.size(), 0)); + } + } + return result; +} + +template +bool CacheAllocator::startNewBackgroundPromoter( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + XDCHECK(numTiers_ > 1); + backgroundPromoter_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundPromoter" + std::to_string(i), backgroundPromoter_[i], interval, strategy); + result = result && ret; + + if (result) { + backgroundPromoter_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundPromoter_.size(), 1)); + } + } + return result; +} + template bool CacheAllocator::stopPoolRebalancer( std::chrono::seconds timeout) { @@ -3800,6 +4072,28 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { return stopWorker("Reaper", reaper_, timeout); } +template +bool CacheAllocator::stopBackgroundEvictor( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundEvictor_.size(); i++) { + auto ret = stopWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], timeout); + result = result && ret; + } + return result; +} + +template +bool CacheAllocator::stopBackgroundPromoter( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundPromoter_.size(); i++) { + auto ret = stopWorker("BackgroundPromoter" + std::to_string(i), backgroundPromoter_[i], timeout); + result = result && ret; + } + return result; +} + template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 81ce90d189..1e6fc79abf 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -36,7 +36,8 @@ #include #include #pragma GCC diagnostic pop - +#include "cachelib/allocator/BackgroundEvictor.h" +#include "cachelib/allocator/BackgroundPromoter.h" #include "cachelib/allocator/CCacheManager.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheAllocatorConfig.h" @@ -695,6 +696,8 @@ class CacheAllocator : public CacheBase { std::shared_ptr resizeStrategy = nullptr, bool ensureProvisionable = false); + auto getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid); + // update an existing pool's config // // @param pid pool id for the pool to be updated @@ -945,6 +948,11 @@ class CacheAllocator : public CacheBase { // @param reaperThrottleConfig throttling config bool startNewReaper(std::chrono::milliseconds interval, util::Throttler::Config reaperThrottleConfig); + + bool startNewBackgroundEvictor(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); + bool startNewBackgroundPromoter(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); // Stop existing workers with a timeout bool stopPoolRebalancer(std::chrono::seconds timeout = std::chrono::seconds{ @@ -954,6 +962,8 @@ class CacheAllocator : public CacheBase { 0}); bool stopMemMonitor(std::chrono::seconds timeout = std::chrono::seconds{0}); bool stopReaper(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundEvictor(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundPromoter(std::chrono::seconds timeout = std::chrono::seconds{0}); // Set pool optimization to either true or false // @@ -988,6 +998,10 @@ class CacheAllocator : public CacheBase { const MemoryPool& getPool(PoolId pid) const override final { return allocator_[currentTier()]->getPool(pid); } + + const MemoryPool& getPoolByTid(PoolId pid, TierId tid) const override final { + return allocator_[tid]->getPool(pid); + } // calculate the number of slabs to be advised/reclaimed in each pool PoolAdviseReclaimData calcNumSlabsToAdviseReclaim() override final { @@ -1034,6 +1048,55 @@ class CacheAllocator : public CacheBase { auto stats = reaper_ ? reaper_->getStats() : ReaperStats{}; return stats; } + + // returns the background evictor + BackgroundEvictionStats getBackgroundEvictorStats() const { + auto stats = BackgroundEvictionStats{}; + for (auto &bg : backgroundEvictor_) + stats += bg->getStats(); + return stats; + } + + BackgroundPromotionStats getBackgroundPromoterStats() const { + auto stats = BackgroundPromotionStats{}; + for (auto &bg : backgroundPromoter_) + stats += bg->getStats(); + return stats; + } + + std::map>> + getBackgroundEvictorClassStats() const { + std::map>> stats; + + for (auto &bg : backgroundEvictor_) { + for (auto &tid : bg->getClassStats()) { + for (auto &pid : tid.second) { + for (auto &cid : pid.second) { + stats[tid.first][pid.first][cid.first] += cid.second; + } + } + } + } + + return stats; + } + + std::map>> + getBackgroundPromoterClassStats() const { + std::map>> stats; + + for (auto &bg : backgroundPromoter_) { + for (auto &tid : bg->getClassStats()) { + for (auto &pid : tid.second) { + for (auto &cid : pid.second) { + stats[tid.first][pid.first][cid.first] += cid.second; + } + } + } + } + + return stats; + } // return the LruType of an item typename MMType::LruType getItemLruType(const Item& item) const; @@ -1181,6 +1244,9 @@ class CacheAllocator : public CacheBase { // gives a relative offset to a pointer within the cache. uint64_t getItemPtrAsOffset(const void* ptr); + bool shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid); + size_t backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers); + // this ensures that we dont introduce any more hidden fields like vtable by // inheriting from the Hooks and their bool interface. static_assert((sizeof(typename MMType::template Hook) + @@ -1222,6 +1288,11 @@ class CacheAllocator : public CacheBase { // allocator and executes the necessary callbacks. no-op if it is nullptr. FOLLY_ALWAYS_INLINE void release(Item* it, bool isNascent); + TierId getTargetTierForItem(PoolId pid, typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime); + // This is the last step in item release. We also use this for the eviction // scenario where we have to do everything, but not release the allocation // to the allocator and instead recycle it for another new allocation. If @@ -1326,7 +1397,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromEvictorThread); // create a new cache allocation on specific memory tier. // For description see allocateInternal. @@ -1337,7 +1409,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromEvictorThread); // Allocate a chained item // @@ -1423,6 +1496,7 @@ class CacheAllocator : public CacheBase { // @return true If the move was completed, and the containers were updated // successfully. bool moveRegularItem(Item& oldItem, ItemHandle& newItemHdl); + bool moveRegularItemForPromotion(Item& oldItem, ItemHandle& newItemHdl); // template class for viewAsChainedAllocs that takes either ReadHandle or // WriteHandle @@ -1577,7 +1651,8 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item); + WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromEvictorThread); + bool tryPromoteToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromEvictorThread); // Try to move the item down to the next memory tier // @@ -1585,7 +1660,11 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(Item& item); + WriteHandle tryEvictToNextMemoryTier(Item& item, bool fromEvictorThread); + bool tryPromoteToNextMemoryTier(Item& item, bool fromEvictorThread); + + bool shouldEvictToNextMemoryTier(TierId sourceTierId, + TierId targetTierId, PoolId pid, Item& item); size_t memoryTierSize(TierId tid) const; @@ -1714,7 +1793,7 @@ class CacheAllocator : public CacheBase { // // @return last handle for corresponding to item on success. empty handle on // failure. caller can retry if needed. - ItemHandle evictNormalItem(Item& item, bool skipIfTokenInvalid = false); + ItemHandle evictNormalItem(Item& item, bool skipIfTokenInvalid = false, bool fromEvictorThread = false); // Helper function to evict a child item for slab release // As a side effect, the parent item is also evicted @@ -1742,6 +1821,133 @@ class CacheAllocator : public CacheBase { folly::annotate_ignore_thread_sanitizer_guard g(__FILE__, __LINE__); allocator_[currentTier()]->forEachAllocation(std::forward(f)); } + + // exposed for the background evictor to iterate through the memory and evict + // in batch. This should improve insertion path for tiered memory config + size_t traverseAndEvictItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t evictions = 0; + size_t evictionCandidates = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + mmContainer.withEvictionIterator([&tries, &candidates, &batch, this](auto &&itr){ + while (candidates.size() < batch && (config_.maxEvictionPromotionHotness == 0 || tries < config_.maxEvictionPromotionHotness) && itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + if (candidate->getRefCount() == 0 && candidate->markMoving()) { + candidates.push_back(candidate); + } + + ++itr; + } + }); + + for (Item *candidate : candidates) { + auto toReleaseHandle = + evictNormalItem(*candidate, true /* skipIfTokenInvalid */, true /* from BG thread */); + auto ref = candidate->unmarkMoving(); + + if (toReleaseHandle || ref == 0u) { + if (candidate->hasChainedItem()) { + (*stats_.chainedItemEvictions)[pid][cid].inc(); + } else { + (*stats_.regularItemEvictions)[pid][cid].inc(); + } + + evictions++; + } else { + if (candidate->hasChainedItem()) { + stats_.evictFailParentAC.inc(); + } else { + stats_.evictFailAC.inc(); + } + } + + if (toReleaseHandle) { + XDCHECK(toReleaseHandle.get() == candidate); + XDCHECK_EQ(1u, toReleaseHandle->getRefCount()); + + // We manually release the item here because we don't want to + // invoke the Item Handle's destructor which will be decrementing + // an already zero refcount, which will throw exception + auto& itemToRelease = *toReleaseHandle.release(); + + // Decrementing the refcount because we want to recycle the item + const auto ref = decRef(itemToRelease); + XDCHECK_EQ(0u, ref); + + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } else if (ref == 0u) { + // it's safe to recycle the item here as there are no more + // references and the item could not been marked as moving + // by other thread since it's detached from MMContainer. + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } + } + + return evictions; + } + + size_t traverseAndPromoteItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t promotions = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + + mmContainer.withPromotionIterator([&tries, &candidates, &batch, this](auto &&itr){ + while (candidates.size() < batch && (config_.maxEvictionPromotionHotness == 0 || tries < config_.maxEvictionPromotionHotness) && itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + // if (candidate->getRefCount() == 0 && candidate->markMoving()) { + // candidates.push_back(candidate); + // } + + // TODO: only allow it for read-only items? + // or implement mvcc + if (!candidate->isExpired() && candidate->markMoving()) { + candidates.push_back(candidate); + } + + ++itr; + } + }); + + for (Item *candidate : candidates) { + auto promoted = tryPromoteToNextMemoryTier(*candidate, true); + auto ref = candidate->unmarkMoving(); + if (promoted) + promotions++; + + if (ref == 0u) { + // stats_.promotionMoveSuccess.inc(); + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } + } + + return promotions; + } // returns true if nvmcache is enabled and we should write this item to // nvmcache. @@ -2050,6 +2256,10 @@ class CacheAllocator : public CacheBase { // free memory monitor std::unique_ptr memMonitor_; + + // background evictor + std::vector>> backgroundEvictor_; + std::vector>> backgroundPromoter_; // check whether a pool is a slabs pool std::array isCompactCachePool_{}; @@ -2105,6 +2315,8 @@ class CacheAllocator : public CacheBase { // Make this friend to give access to acquire and release friend ReadHandle; friend ReaperAPIWrapper; + friend BackgroundEvictorAPIWrapper; + friend BackgroundPromoterAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index ca51deb94c..d6b9bc354a 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -32,6 +32,7 @@ #include "cachelib/allocator/NvmAdmissionPolicy.h" #include "cachelib/allocator/PoolOptimizeStrategy.h" #include "cachelib/allocator/RebalanceStrategy.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" #include "cachelib/allocator/Util.h" #include "cachelib/common/EventInterface.h" #include "cachelib/common/Throttler.h" @@ -266,6 +267,16 @@ class CacheAllocatorConfig { std::chrono::seconds regularInterval, std::chrono::seconds ccacheInterval, uint32_t ccacheStepSizePercent); + + // Enable the background evictor - scans a tier to look for objects + // to evict to the next tier + CacheAllocatorConfig& enableBackgroundEvictor( + std::shared_ptr backgroundEvictorStrategy, + std::chrono::milliseconds regularInterval, size_t threads); + + CacheAllocatorConfig& enableBackgroundPromoter( + std::shared_ptr backgroundEvictorStrategy, + std::chrono::milliseconds regularInterval, size_t threads); // This enables an optimization for Pool rebalancing and resizing. // The rough idea is to ensure only the least useful items are evicted when @@ -337,6 +348,17 @@ class CacheAllocatorConfig { compactCacheOptimizeInterval.count() > 0) && poolOptimizeStrategy != nullptr; } + + // @return whether background evictor thread is enabled + bool backgroundEvictorEnabled() const noexcept { + return backgroundEvictorInterval.count() > 0 && + backgroundEvictorStrategy != nullptr; + } + + bool backgroundPromoterEnabled() const noexcept { + return backgroundPromoterInterval.count() > 0 && + backgroundPromoterStrategy != nullptr; + } // @return whether memory monitor is enabled bool memMonitoringEnabled() const noexcept { @@ -433,6 +455,13 @@ class CacheAllocatorConfig { // time interval to sleep between iterators of rebalancing the pools. std::chrono::milliseconds poolRebalanceInterval{std::chrono::seconds{1}}; + + // time interval to sleep between runs of the background evictor + std::chrono::milliseconds backgroundEvictorInterval{std::chrono::milliseconds{1000}}; + std::chrono::milliseconds backgroundPromoterInterval{std::chrono::milliseconds{1000}}; + + size_t backgroundEvictorThreads{1}; + size_t backgroundPromoterThreads{1}; // Free slabs pro-actively if the ratio of number of freeallocs to // the number of allocs per slab in a slab class is above this @@ -444,6 +473,10 @@ class CacheAllocatorConfig { // rebalance to avoid alloc fialures. std::shared_ptr defaultPoolRebalanceStrategy{ new RebalanceStrategy{}}; + + // rebalance to avoid alloc fialures. + std::shared_ptr backgroundEvictorStrategy; + std::shared_ptr backgroundPromoterStrategy; // time interval to sleep between iterations of pool size optimization, // for regular pools and compact caches @@ -585,6 +618,33 @@ class CacheAllocatorConfig { // skip promote children items in chained when parent fail to promote bool skipPromoteChildrenWhenParentFailed{false}; + bool disableEvictionToMemory{false}; + + double promotionAcWatermark{4.0}; + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + double minAcAllocationWatermark{0.0}; + double maxAcAllocationWatermark{0.0}; + uint64_t sizeThresholdPolicy{0}; + double defaultTierChancePercentage{50.0}; + // TODO: default could be based on ratio + + double numDuplicateElements{0.0}; // inclusivness of the cache + double syncPromotion{0.0}; // can promotion be done synchronously in user thread + + uint64_t evictorThreads{1}; + uint64_t promoterThreads{1}; + + uint64_t maxEvictionBatch{40}; + uint64_t maxPromotionBatch{10}; + + uint64_t minEvictionBatch{1}; + uint64_t minPromotionBatch{1}; + + uint64_t maxEvictionPromotionHotness{60}; + + uint64_t forceAllocationTier{UINT64_MAX}; + friend CacheT; private: @@ -933,6 +993,26 @@ CacheAllocatorConfig& CacheAllocatorConfig::enablePoolRebalancing( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundEvictor( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t evictorThreads) { + backgroundEvictorStrategy = strategy; + backgroundEvictorInterval = interval; + backgroundEvictorThreads = evictorThreads; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundPromoter( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t promoterThreads) { + backgroundPromoterStrategy = strategy; + backgroundPromoterInterval = interval; + backgroundPromoterThreads = promoterThreads; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enablePoolResizing( std::shared_ptr resizeStrategy, diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index f82ba143e3..c8af1a2a98 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -300,6 +300,43 @@ struct ReaperStats { uint64_t avgTraversalTimeMs{0}; }; +// Eviction Stats +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t numEvictedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + + // total number of classes + uint64_t totalClasses{0}; + + // eviction size + uint64_t evictionSize{0}; + + BackgroundEvictionStats& operator+=(const BackgroundEvictionStats& rhs) { + numEvictedItems += rhs.numEvictedItems; + runCount += rhs.runCount; + totalClasses += rhs.totalClasses; + evictionSize += rhs.evictionSize; + return *this; + } +}; + +struct BackgroundPromotionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t numPromotedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + + BackgroundPromotionStats& operator+=(const BackgroundPromotionStats& rhs) { + numPromotedItems += rhs.numPromotedItems; + runCount += rhs.runCount; + return *this; + } +}; + // CacheMetadata type to export struct CacheMetadata { // allocator_version @@ -320,6 +357,11 @@ struct Stats; // Stats that apply globally in cache and // the ones that are aggregated over all pools struct GlobalCacheStats { + // background eviction stats + BackgroundEvictionStats evictionStats; + + BackgroundPromotionStats promotionStats; + // number of calls to CacheAllocator::find uint64_t numCacheGets{0}; diff --git a/cachelib/allocator/FreeThresholdStrategy.cpp b/cachelib/allocator/FreeThresholdStrategy.cpp new file mode 100644 index 0000000000..5ffc718fa7 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/FreeThresholdStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + + + +FreeThresholdStrategy::FreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t maxEvictionBatch, uint64_t minEvictionBatch) + : lowEvictionAcWatermark(lowEvictionAcWatermark), highEvictionAcWatermark(highEvictionAcWatermark), maxEvictionBatch(maxEvictionBatch), minEvictionBatch(minEvictionBatch) {} + +std::vector FreeThresholdStrategy::calculateBatchSizes( + const CacheBase& cache, std::vector> acVec) { + std::vector batches{}; + for (auto [tid, pid, cid] : acVec) { + auto stats = cache.getAllocationClassStats(tid, pid, cid); + if (stats.approxFreePercent >= highEvictionAcWatermark) { + batches.push_back(0); + } else { + auto toFreeMemPercent = highEvictionAcWatermark - stats.approxFreePercent; + auto toFreeItems = static_cast(toFreeMemPercent * stats.memorySize / stats.allocSize); + batches.push_back(toFreeItems); + } + } + + if (batches.size() == 0) { + return batches; + } + + auto maxBatch = *std::max_element(batches.begin(), batches.end()); + if (maxBatch == 0) + return batches; + + std::transform(batches.begin(), batches.end(), batches.begin(), [&](auto numItems){ + if (numItems == 0) { + return 0UL; + } + + auto cappedBatchSize = maxEvictionBatch * numItems / maxBatch; + if (cappedBatchSize < minEvictionBatch) + return minEvictionBatch; + else + return cappedBatchSize; + }); + + return batches; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/FreeThresholdStrategy.h b/cachelib/allocator/FreeThresholdStrategy.h new file mode 100644 index 0000000000..6a6b0c8950 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class FreeThresholdStrategy : public BackgroundEvictorStrategy { + +public: + FreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t maxEvictionBatch, uint64_t minEvictionBatch); + ~FreeThresholdStrategy() {} + + std::vector calculateBatchSizes(const CacheBase& cache, + std::vector> acVecs); +private: + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + uint64_t maxEvictionBatch{40}; + uint64_t minEvictionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/MM2Q-inl.h b/cachelib/allocator/MM2Q-inl.h index 2f1d538612..be87a4a093 100644 --- a/cachelib/allocator/MM2Q-inl.h +++ b/cachelib/allocator/MM2Q-inl.h @@ -14,6 +14,8 @@ * limitations under the License. */ +#include + namespace facebook { namespace cachelib { @@ -104,6 +106,10 @@ bool MM2Q::Container::recordAccess(T& node, return false; } + // TODO: % 100 is not very accurate + if (config_.markUsefulChance < 100.0 && folly::Random::rand32() % 100 >= config_.markUsefulChance) + return false; + return lruMutex_->lock_combine(func); } return false; @@ -211,15 +217,32 @@ void MM2Q::Container::rebalance() noexcept { template T::*HookPtr> bool MM2Q::Container::add(T& node) noexcept { const auto currTime = static_cast