509318396 ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 yb@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZmZddlmZddlmZddlTddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZdd lm Z ddlm!Z!ddlm"Z"ddlm#Z#ddl$m%Z%m&Z&ddlm'Z'ddl(m)Z)ddl*m+Z+m,Z,ddl-m.Z.e%e/Z$dZ0e+eZ1dP\Z2Z3Z4Z5dZ6dZ7ddZ8Gdd d eZ9Gd!d"d"e9Z:Gd#d$d$eZ;Gd%d&d&eZdQdRfffdSdTdUdVdWffdXdYfdZd;Z?dd?d?ZAGd@dAdAZBdBdCZCGdDdEdEZDdFdGZEGdHdIdIZFGdJdKdKeFZGdLddddLdMdNdOZHdS)[N)PopenPIPE) __version__)LZ4)*)ErrorIntegrityError) bin_to_hex) get_base_dir)get_limited_unpacker)hostname_is_unique)replace_placeholders)sysinfo)format_file_size) safe_unlink)prepare_subprocess_env) create_logger setup_logging)msgpack) Repository) parse_versionformat_version)xxh64imardg?cCsNt|}}x<|rHtj||}||8}|s,P||d}tj|dqW|S)z;os.write wrapper so we do not lose data for partial writes.Ng& .>)lenoswritetimesleep)fddataZamountZ remainingcountr(/usr/lib64/python3.6/remote.pyos_write1s   r*c@seZdZdZdS)ConnectionClosedz Connection closed by remote hostN)__name__ __module__ __qualname____doc__r(r(r(r)r+Dsr+c@seZdZdZdS)ConnectionClosedWithHintz$Connection closed by remote host. {}N)r,r-r.r/r(r(r(r)r0Hsr0c@seZdZdZdS)PathNotAllowedzRepository path not allowed: {}N)r,r-r.r/r(r(r(r)r1Lsr1c@seZdZdZdS)InvalidRPCMethodzRPC method {} is not validN)r,r-r.r/r(r(r(r)r2Psr2c@seZdZdZdS)!UnexpectedRPCDataFormatFromClientz4Borg {}: Got unexpected RPC data format from client.N)r,r-r.r/r(r(r(r)r3Tsr3cs eZdZdZfddZZS)!UnexpectedRPCDataFormatFromServerz.Got unexpected RPC data format from server: {}c sfy|jdd}Wn@tk rT|dd}dd|D}tjdj|d}YnXtj|dS)NcSsg|] }d|qS)z%02Xr().0Zbyter(r(r) `sz>UnexpectedRPCDataFormatFromServer.__init__.. 0)decodeUnicodeDecodeErrortextwrapZfilljoinsuper__init__)selfr&) __class__r(r)rA[s z*UnexpectedRPCDataFormatFromServer.__init__)r,r-r.r/rA __classcell__r(r()rCr)r4Xsr4repair save_spacelimitmarkeridr&keydata client_datapathcreate lock_waitlock exclusive append_onlynext_unreserved start_nonce)checkcommitrollbackdestroy__len__listputgetdeletesave_keyload_key break_lock negotiateopenget_free_noncecommit_nonce_reservationcsfddDS)Ncsi|]}||jqSr()r<)r6k)dr(r) szdecode_keys..r()rer()rer) decode_keyssrgc@sReZdZd&ZddZddZddZddZddZddZ d'd"d#Z d$d%Z d S)(RepositoryServerrXrTrUr\rWr[rYscanr`rarZrVr]r^r_rbrcinject_exceptioncCs,d|_||_||_||_||_td|_dS)Nz1.0.8) repositoryrestrict_to_pathsrestrict_to_repositoriesrQ storage_quotarclient_version)rBrlrmrQrnr(r(r)rAs zRepositoryServer.__init__c snyfddtt|DStk rh|dkrPtdkrPt|dd}nfddt|DSXdS)z5Translate from positional protocol to named protocol.csi|]\}}||qSr(r()r6posname)argvr(r)rfsz8RepositoryServer.positional_to_named..raNcsi|]\}}||qSr(r()r6rprq)rrr(r)rfs) enumerate compatMap IndexErrorr )rBmethodrrmappingr()rrr)positional_to_namedsz$RepositoryServer.positional_to_namedcs$ttj|jfddDS)zWRemove unknown named parameters from call, because client did (implicitly) say it's ok.csi|]}|kr||qSr(r()r6rq)knownkwargsr(r)rfsz0RepositoryServer.filter_args..)setinspect signature parameters)rBfr{r()rzr{r) filter_argsszRepositoryServer.filter_argsc"Cstjj}tjj}tjj}tj|tj}tj|tj|tjBtj|tj}tj|tj|tj@tj|tj}tj|tj|tj@t d}xvt j |gggd\}}}|rtj |t } | s|j dk r|j jnt|djtjdS|j| x|D]} t| tr@d} | t} | tj} t| t}n^t| trt| dkrd} | \}} } }| j} |j| |}n|j dk r|j jttyb| |jkrt | yt!|| }Wn"t"k rt!|j | }YnX|j#||}|f|}Wnt$k r}z| rt%j&|j'|}t%j(tj)}d}t|t*rX|j+g}|j%}t|t,j-t,j.t/frpnt0j1dj2|y2t3j4t| d|j'j5d |j6d |d |d |d t7i}WnNt8k rt3j4t| d|j'j5d dd|j6Dd |d |d |d t7i}YnXt||nt|t,j-t,j.t/fr&ndt|t*rP|j%r@t0j9nt0j:}|j+}nt0j9}d|j'j5}dt%j;t7f}t0j<|t0j=||d}t|t3j4d| |j'j5|fWYdd}~Xn:X| rt|t3j4t| t>|int|t3j4d| d|fqW|r|j jdSqWdS)NZserver z.z%s Exception in RPC callz%s %sz3Remote Exception (see remote log for the traceback)r)?sysstdinfilenostdoutfcntlF_GETFLF_SETFLr! O_NONBLOCKr selectreadBUFSIZErkcloser*formatrencodefeedrdictMSGIDMSGr<rgARGStupler ryr3 rpc_methodsr2getattrAttributeErrorr BaseException tracebackformat_exception_onlyrCformat_exceptionexc_infor get_messager DoesNotExist AlreadyExistsr1loggingdebugr?rpackbr,argsr TypeErrorERRORDEBUG format_excerrorlogRESULT)rBstdin_fd stdout_fd stderr_fdZflunpackerrwZesr&unpacked dictFormatmsgidrwr_rreseZex_shortZex_fullZex_tracemsgZ tb_log_leveltbexcr(r(r)serves                       . zRepositoryServer.servecCsZ|tkr tSt|trL|d|_tjtjdj}tdd|dt j dnt |_dt iS)Nsclient_versionT)Zis_servejsonlevelz2Initialized logging system for JSON-based protocolserver_version) RPC_PROTOCOL_VERSIONrrrorZ getLevelName getLoggerrrloggerr BORG_VERSION)rBrKrr(r(r)r`6s   zRepositoryServer.negotiatecCst|trtj|}|jdr8tjjt|dd}nr|jdr\tjjt|dd}nN|jdr|tjj|dd}n.|jdrtjj|}n|jdr|dd}tjj |S) Nz/~/r:z~/rz/~r~z/./) rrr!fsdecode startswithrLr?r expanduserrealpath)rBrLr(r(r) _resolve_pathGs        zRepositoryServer._resolve_pathFNTc Cstjd||j|}tjd|tjj|d}|jrpx8|jD]&} tjjtjj| d} |j| r>Pq>Wt ||j rx6|j D]$} tjjtjj| d} | |kr~Pq~Wt || r|j p|}t ||||||j ||d|_|jj|jjS)NzResolving repository path %rzResolved repository path to %rr)rNrOrQrnrPmake_parent_dirs)rrrr!rLr?rlrrr1rmrQrrnrk __enter__rI) rBrLrMrNrOrPrQrZ path_with_sepZrestrict_to_pathZrestrict_to_path_with_sepZrestrict_to_repositoryZrestrict_to_repository_with_sepr(r(r)raWs0         zRepositoryServer.opencCs|j}d}d}|dkr$tj|n|dkr8tj|np|dkrLtj|n\|dkr^t|nJ|dkrptdn8|d krtj||n"|d krt|n|d krd d dS) Nz test stringz test string2rr CheckNeededr r1ZfooObjectNotFoundr2Zdivider) r<rrrrr r1rr2)rBZkinds1s2r(r(r)rjzs&      z!RepositoryServer.inject_exception)rXrTrUr\rWr[rYrir`rarZrVr]r^r_rbrcrj)FNTNFF) r,r-r.rrAryrrr`rrarjr(r(r(r)rhs6  q "rhc@seZdZddZddZdS)SleepingBandwidthLimitercCs0|r&t|t|_tj|_|j|_nd|_dS)N)rRATELIMIT_PERIOD ratelimitr# monotonicratelimit_lastratelimit_quota)rBrGr(r(r)rAs   z!SleepingBandwidthLimiter.__init__cCs|jrtj}|jt|krN|j|j7_|jd|jkrHd|j|_||_|jdkr|jt|}tj||j|j7_tj|_t||jkr|d|j}tj ||}|jr|j|8_|S)Nrr) rr#rrrrr$r r!r")rBr%to_sendZnowZtosleepwrittenr(r(r)r"s$     zSleepingBandwidthLimiter.writeN)r,r-r.rAr"r(r(r(r)rsrc sfdd}|S)a~Check version requirements and use self.call to do the remote method call. specifies the version in which borg introduced this method, calling this method when connected to an older version will fail without transmiting anything to the server. Further kwargs can be used to encode version specific restrictions. If a previous hardcoded behaviour is parameterized in a version, this allows calls that use the previously hardcoded behaviour to pass through and generates an error if another behaviour is requested by the client. e.g. when 'append_only' was introduced in 1.0.7 the previous behaviour was what now is append_only=False. Thus @api(..., append_only={'since': parse_version('1.0.7'), 'previously': False}) allows calls with append_only=False for all version but rejects calls using append_only=True on versions older than 1.0.7. cstjfdd}|S)Nc s tj}|j|f||}i}i}xh|jjD]Z\}}|dkrBq0||jkrt|dkrd|j|||<q|j|||<q0|j|jk r0|j||<q0W|jkr|j j t xbjD]V\}} | d|jkrqd| kr||| dkrq|j dj j |||t | dqW|j j |f|S)NrBwaitsince previouslyz {0} {1}={2!s})r}r~ZbindritemsZ argumentsdefaultemptyrRPCServerOutdatedr,rrcall) rBrr{ZsigZ bound_argsZnamedZextrarqZparamZ restriction)rkwargs_decoratorrr(r)do_rpcs.    z&api..decorator..do_rpc) functoolswraps)rr)rr)rr) decoratorszapi..decoratorr()rrrr()rrr)apis!rc@s0eZdZgZGdddeZGdddeZdZdMdd Z d d Z d d Z ddZ ddZ eddZddZddZddZddZdNddZeededdd ed!dd d"dOd#d$Zeedd%dPd&d'Zeedd%dQd(d)Zeedd%d*d+Zeedd%d,d-Zeedd%d.d/Zeedd%dRd0d1Zeed2d%dSd3d4Zd5d6ZdTd7d8Z eedd%dUd9d:Z!eedd%dVd;d<Z"eedd%d=d>Z#eedd%d?d@Z$eedd%dAdBZ%eedd%dCdDZ&eedd%dEdFZ'dGdHZ(dWdIdJZ)dKdLZ*dS)XRemoteRepositoryc@sLeZdZddZddZeddZeddZed d Zed d Z d S)zRemoteRepository.RPCErrorcCs ||_dS)N)r)rBrr(r(r)rAsz"RemoteRepository.RPCError.__init__cCs(d|jkrdj|jdjS|jSdS)Nsexception_short )rr?r<exception_class)rBr(r(r)rs z%RemoteRepository.RPCError.get_messagecCs|jjddS)Nsexception_traceT)rr[)rBr(r(r)rsz#RemoteRepository.RPCError.tracebackcCs|jdjS)Nsexception_class)rr<)rBr(r(r)rsz)RemoteRepository.RPCError.exception_classcCs.d|jkrdj|jdjS|jdSdS)Nsexception_fullrz4 Remote Exception (see remote log for the traceback))rr?r<r)rBr(r(r)exception_fulls z(RemoteRepository.RPCError.exception_fullcCs d|jkr|jdjSdSdS)Nssysinfor)rr<)rBr(r(r)rs z!RemoteRepository.RPCError.sysinfoN) r,r-r.rArpropertyrrrrr(r(r(r)RPCErrors    rc@s(eZdZdZeddZeddZdS)z"RemoteRepository.RPCServerOutdatedz2Borg server is too old for {}. Required version {}cCs |jdS)Nr)r)rBr(r(r)rw sz)RemoteRepository.RPCServerOutdated.methodcCs |jdS)Nr)r)rBr(r(r)required_versionsz3RemoteRepository.RPCServerOutdated.required_versionN)r,r-r.r/rrwrr(r(r(r)rs rFNTc s|__g_d_d_d_d_d_i_t _ i_ i_ d_ t|rf|jrf|jdnd_td_td_d_|_|jdk} t| d} j|| } | sˆj|| } tjd| t| dttt| d _jjj _!jj"j _#jj$j _%t&j&j!t&j't&j&j!t&j(t)j*Bt&j&j#t&j't&j&j#t&j(t)j*Bt&j&j%t&j't&j&j%t&j(t)j*Bj#j%g_+j!j#j%g_,yyj-d d d t.ii} Wn t/k rt0d dYnX| t1krd_2n4t3| t4rd| krd_2| d_n t5d| fdd} j2rF| nhy | Wn\j6k r}z<|j7dkrtd}t8j$j9|td_dt:d<| WYdd}~XnXWn t5k rj;YnXdS)NriZclientz1.0.8Z __testsuite__)systemzSSH command line: %s)bufsizerrstderrenvr`rKsclient_versionzIs borg working on the server?Fsserver_versionTz8Server insisted on using unsupported protocol version %sc s"jjjd_dS)N)rLrMrNrOrPrQr)ralocationrLrIr()rQrMrPrOrNrrBr(r)do_openIsz*RemoteRepository.__init__..do_openraoPlease note: If you see a TypeError complaining about the number of positional arguments given to open(), you can ignore it if it comes from a borg version < 1.0.7. This TypeError is a cosmetic side effect of the compatibility code borg clients >= 1.0.7 have to support older borg servers. This problem will go away as soon as the server has been upgraded to 1.0.7+. z1.0.6rLrMrNrOra)rLrMrNrO)t|jr tjdt|jf|jr:|jds:tddS)Nz2still %d cached responses left in RemoteRepositoryFz&cleanup happened in Repository.__del__)r rrrrrAssertionError)rBr(r(r)__del__is  zRemoteRepository.__del__cCsd|jj|jjfS)Nz<%s %s>)rCr,rZcanonical_path)rBr(r(r)__repr__pszRemoteRepository.__repr__cCs|S)Nr()rBr(r(r)rsszRemoteRepository.__enter__cCsPz"|dk r tjd|_|jWdtjdt|jt|j|j |j XdS)NzDRemoteRepository: %s bytes sent, %s bytes received, %d messages sent) r#rrrVrrrrrrr)rBexc_typeexc_valexc_tbr(r(r)__exit__vs zRemoteRepository.__exit__cCs t|jS)N)r rI)rBr(r(r)id_strszRemoteRepository.id_strcCsXg}|dk r|jd|jtj}|jtjr<|jdn^|jtjrT|jdnF|jtjrbn8|jtjrz|jdn |jtj r|jdnt dx4|j D]*}d|krd |}d |kr|jd |qWd |kr|j r|jd |j g}t s|jd|r&|tjdddg||jS|jp:tjjdd}t|}||dg|SdS)z return a borg serve command lineNz --umask=%03oz--debugz--infoz--errorz --criticalz log level missing, fix this code.z borg.debug.rkz--debug-topic=%srnz--storage-quota=%szBORG_HOSTNAME_IS_UNIQUE=noz-mz borg.archiverrZBORG_REMOTE_PATHborg)appendumaskrrZ isEnabledForrINFOZWARNINGrCRITICAL ValueErrorZ debug_topicsrnr r executableextra_test_args remote_pathr!environr[r)rBrr ZoptsZ root_loggerZtopicZenv_varsr!r(r(r)rs<           zRemoteRepository.borg_cmdcCsj|jjptjjdd}tj|}|jr8|dt|jg7}|j rV|j d|j |j fn|j d|j |S)zEreturn a ssh command line that can be prefixed to a borg command lineZBORG_RSHZsshz-pz%s@%sz%s) rrshr!r"r[shlexsplitZportruserrr)rBrr#rr(r(r)rs zRemoteRepository.ssh_cmdcsfddt|DS)Ncsg|] }|qSr(r()r6rq)r{r(r)r7sz8RemoteRepository.named_to_positional..)ru)rBrwr{r()r{r)named_to_positionalsz$RemoteRepository.named_to_positionalcKs"x|j||gf|D]}|SWdS)N) call_many)rBcmdrkwrespr(r(r)rszRemoteRepository.callc#s| r|dkrdSfdd}fdd}t|}g}x>|sD|rvjrttjjkrttjdt|tjdSxj|ryJjj |d} |j dd| kr|| n| t V| r| rdSWqvt k rPYqvXqvW|dkrNxbyjj \} } Wn,t k r*|r"j r"PndSYqXd| kr@|| q| t VqWjsr|sdjr|t|tkr|jg} ng} tjj| jd \} } }|rtd x| D]}|jkrtj|t}|stjt|7_jj|xȈjD]} t| tr| t } nPt| t!rbt| d krb| \}} }}|rTt | d|i} n t | t |i} nt"|| j krj j#| d| kr| j| <n| t dk r| j| <n | j| <qWn|j$krtj|d }|stjt|7_j%rj%|}d _%|j&dd}|r@|dj'd r@|j _%x|D]}t(|j)dqFWqW| r:xj r|sjrt|tkr|rr|r|dkst*d|ddj+krr|j,||j ddn|j d}|dkr|dj+kr|j,||dn\j-d 7_-|j,j-j.rTt/j0t j-t1|t2|i_nt/j0d j-|j3||f_j rljrljj d}d|i}j-d 7_-j+j4|gj,j-j.rt/j0t j-t1dt2|i_nt/j0d j-dj3d|f_qlWjr:y4j5j6jj}j7|7_7j|d_Wq:t8k rr}z|j9t9j:krbWYdd}~Xq:Xq:Wj t;|O_ dS)Nrcs&j|jd}j|s"j|=|S)Nr)rpop)Zchunkidr)rBr(r)pop_preload_msgids z5RemoteRepository.call_many..pop_preload_msgidcs||dj}d|k}|jd}|dkr8tjjjn@|dkrRtjjjn&|dkrltjjjn |dkr|rtdnt|djn|d kr|rtj dntj |djn|d kr|rt d nt |djn|d krtj |djnr|d krB|r(tj djjntj |djjjn6|dkrn|r\t dnt |djn j|dS)Nsexception_classsexception_argsrrrr z(not available)rAtticRepositoryr1z (unknown)ParentPathDoesNotExistrr2)r<r[rrrZ processedrrr r.r1r/rr2r)rrZ old_serverr)rBr(r) handle_errors@         z0RemoteRepository.call_many..handle_errorzPshutdown_time reached, shutting down with %d waiting_for and %d async_responses.rsexception_classrzFD exception occurredrsirT)keepends rzutf-8r[z&is_preload is only supported for 'get'rI)r2r).)r?)r()rBidsr?r+r(r(r)rCs zRemoteRepository.get_manycCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBrIr&rr(r(r)rZszRemoteRepository.putcCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBrIrr(r(r)r\szRemoteRepository.deletecCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBrJr(r(r)r]szRemoteRepository.save_keycCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBr(r(r)r^szRemoteRepository.load_keycCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBr(r(r)rbszRemoteRepository.get_free_noncecCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBrRrSr(r(r)rcsz)RemoteRepository.commit_nonce_reservationcCsdS)z;actual remoting is done via self.call in the @api decoratorNr()rBr(r(r)r_szRemoteRepository.break_lockcCs2|jr.|jjj|jjj|jjd|_dS)N)rrrrr)rBr(r(r)rs    zRemoteRepository.closecCs"x|jdgd|dD]}|SWdS)NrT)r>rr@)r()rBrr+r(r(r)async_responseszRemoteRepository.async_responsecCs|j|7_dS)N)r)rBrDr(r(r)preloadszRemoteRepository.preload)FFNTFFN)TFT)FNTFFF)FF)F)NN)NN)F)T)T)T)+r,r-r.r r rrrrrArrrrrrrrr'rr(rrrarTrUrVrWrXrYrir[rCrZr\r]r^rbrcr_rrErFr(r(r(r)rs^"  S 1  7                rcCs|jdst|jdrbtj|}|ddkrFtjd|d|d S|ddkrtt|d tj }t |t srttj |d }d |d |d <tj djr||j krtjjtj|dn|j|d|d n|djdr tj d}|j tjkr tj djr>d |kr&d |d |d <tjjtj|dn"d |kr tjjd |d dn|jdr|jdd\}}}tt|tj }|jdrtj||jn&|jdd\}}tj |j||jn8tj djrtj djd |jntjjd |d S)zv Handle a remote log line. This function is remarkably complex because it handles multiple wire formats.  r{typeprogress_messageprogress_percent log_messagez3Dropped remote log message with unknown type %r: %sNZ levelnamerqzRemote: messagerz%sZ progress_zborg.output.progressz$LOG r8rzRemote:rr)rGr)rJrKrL)r9rrrloadsrZwarningrrrrrrZgetEffectiveLevelrrr"dumpsrrr%rstripstrip)rBrrZ target_loggerZprogress_loggerrZlognamer(r(r)r:sB         r:c@sLeZdZdZdddZddZddZd d Zd d ZdddZ ddZ dS)RepositoryNoCacheaRA not caching Repository wrapper, passes through to repository. Just to have same API (including the context manager) as RepositoryCache. *transform* is a callable taking two arguments, key and raw repository data. The return value is returned from get()/get_many(). By default, the raw repository data is returned. NcCs||_|pdd|_dS)NcSs|S)Nr()keyr&r(r(r),sz,RepositoryNoCache.__init__..)rk transform)rBrkrUr(r(r)rA*szRepositoryNoCache.__init__cCsdS)Nr()rBr(r(r)r.szRepositoryNoCache.closecCs|S)Nr()rBr(r(r)r1szRepositoryNoCache.__enter__cCs |jdS)N)r)rBrrrr(r(r)r4szRepositoryNoCache.__exit__cCst|j|gddS)NF)cache)nextrC)rBrSr(r(r)r[7szRepositoryNoCache.getTccs2x,t||jj|D]\}}|j||VqWdS)N)ziprkrCrU)rBkeysrVrSr&r(r(r)rC:szRepositoryNoCache.get_manycCsdS)Nr()rBr(r(r)log_instrumentation>sz%RepositoryNoCache.log_instrumentation)N)T) r,r-r.r/rArrrr[rCrZr(r(r(r)rR!s  rRcs\eZdZdZdfdd ZddZddZd d Zd d Zd dZ ddZ dddZ Z S)RepositoryCacheaf A caching Repository wrapper. Caches Repository GET operations locally. *pack* and *unpack* complement *transform* of the base class. *pack* receives the output of *transform* and should return bytes, which are stored in the cache. *unpack* receives these bytes and should return the initial data (as returned by *transform*). Ncsvtj|||pdd|_|p&dd|_t|_tjdd|_|j d|_ d|_ d|_ d|_ d|_d|_d|_dS)NcSs|S)Nr()r&r(r(r)rTPsz*RepositoryCache.__init__..cSs|S)Nr()r&r(r(r)rTQsz borg-cache-)prefixrg)r@rApackunpackr|rVtempfileZmkdtempbasedirquery_size_limitsizehitsmisses slow_missesslow_lat evictionsenospc)rBrkr]r^rU)rCr(r)rANszRepositoryCache.__init__cCs0tj|j}|j|j}tt|dd|_dS)Ng?rl)r!statvfsr`f_bavailf_frsizermin size_limit)rBZstat_fsZavailable_spacer(r(r)ra^s  z RepositoryCache.query_size_limitcCstjj|jt|S)N)r!rLr?r`r )rBrSr(r(r) key_filenamecszRepositoryCache.key_filenamecCsr|jtd|j}xV|j|krl|jrl|jj}|j|}|jtj|j 8_tj ||j d7_ qWdS)Ng?r) rarrnrbrVr,ror!statst_sizeunlinkrg)rBZ target_sizerSfiler(r(r)backofffs   zRepositoryCache.backoffc Cs|j||}|s|S|j|}|j|}y$t|d}|j|WdQRXWnjtk r}zNy t|Wntk r~YnX|jtj kr|j d7_ |j nWYdd}~Xn4X|j t |7_ |jj||j |jkr|j |S)Nwbr)rUr]rorar"r<rFileNotFoundErrorr=ZENOSPCrhrtrbr rVaddrn) rBrSr&rV transformedZpackedrsr%Zos_errorr(r(r) add_entryps,         zRepositoryCache.add_entryc Cs>tjdt|jt|jt|j|j|j|j |j |j |j dS)NzyRepositoryCache: current items %d, size %s / %s, %d hits, %d misses, %d slow misses (+%.1fs), %d evictions, %d ENOSPC hit) rrr rVrrbrnrcrdrerfrgrh)rBr(r(r)rZsz#RepositoryCache.log_instrumentationcCs"|j|jjtj|jdS)N)rZrVclearshutilZrmtreer`)rBr(r(r)rs zRepositoryCache.closeTc #sfdd|D}t|jj|}x|D]}|jkrxj|}t|d$}jd7_j|jVWdQRXq*x|D]4\}} ||kr~j || |} j d7_ | VPq~Wt j } jj |} jt j | 7_j || |} jd7_| Vq*Wx|D]} q WdS)Ncsg|]}|jkr|qSr()rV)r6rS)rBr(r)r7sz,RepositoryCache.get_many..rbr)rXrkrCrVrorarcr^rryrdr#Z perf_counterr[rfre) rBrYrVZ unknown_keysZrepository_iteratorrSrsr%Zkey_r&rxZt0rr()rBr)rCs,       zRepositoryCache.get_many)NNN)T) r,r-r.r/rArarortryrZrrCrDr(r()rCr)r[Bs  r[F)decrypted_cacher]r^rU force_cachecs|r|s|s|rtdn@|rZ|tjdtfdd}fdd}fdd}t|tsh|rvt||||St||Sd S) a Return a Repository(No)Cache for *repository*. If *decrypted_cache* is a key object, then get and get_many will return a tuple (csize, plaintext) instead of the actual data in the repository. The cache will store decrypted data, which increases CPU efficiency (by avoiding repeatedly decrypting and more importantly MAC and ID checking cached objects). Internally, objects are compressed with LZ4. z:decrypted_cache and pack/unpack/transform are incompatiblez=I8scs&|\}}j|}j|t||S)N)compressr]r)r&csize decrypted compressed) cache_struct compressorr(r)r]s zcache_if_remote..packcsPt|}j|dj\}}|jd}|t|krBtd|j|fS)Nz)detected corrupted data in metadata cache) memoryviewr^rbrr Z decompress)r&rZchecksumr)rrr(r)r^s  zcache_if_remote..unpackcst|}j||}||fS)N)r Zdecrypt)Zid_r&rr)rSr(r)rUs z"cache_if_remote..transformN)rstructStructrrrr[rR)rkr}r]r^rUr~r()rrrSr)cache_if_remotes    r)rrrr)rErF)rF)rGrH)rIr&)rI)rI)rJ)rK)rLrMrNrOrPrQ)rRrS)Ir=rrr}rrr!rr$r{rrr_r>r#r subprocessrrrrrrZ constantsZhelpersrr r r r r rrrrrrrrrrkrr rrZalgorithms.checksumsrr,rrrrrrr6rr*r+r0r1r2r3r4rurgrhrrrr:rRr[rr(r(r(r)s               #4xJ!q