diff --git a/docs/latest/.buildinfo b/docs/latest/.buildinfo index fdb980b3..72d0c258 100644 --- a/docs/latest/.buildinfo +++ b/docs/latest/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: f74829a8d80ac23fb5f1ba14167f300f +config: 4d96852847ec225f6bda6ce4c0135d77 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/latest/google.resumable_media.requests.download.html b/docs/latest/google.resumable_media.requests.download.html index c7e99fc4..9b00c969 100644 --- a/docs/latest/google.resumable_media.requests.download.html +++ b/docs/latest/google.resumable_media.requests.download.html @@ -331,6 +331,8 @@
consume(transport)ΒΆ

Consume the resource to be downloaded.

+

If a stream is attached to this download, then the downloaded +resource will be written to the stream.

diff --git a/docs/latest/searchindex.js b/docs/latest/searchindex.js index f6b6f553..cfbed4d5 100644 --- a/docs/latest/searchindex.js +++ b/docs/latest/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["google.resumable_media.common","google.resumable_media.requests","google.resumable_media.requests.download","google.resumable_media.requests.upload","index"],envversion:53,filenames:["google.resumable_media.common.rst","google.resumable_media.requests.rst","google.resumable_media.requests.download.rst","google.resumable_media.requests.upload.rst","index.rst"],objects:{"google.resumable_media":{common:[0,0,0,"-"],requests:[1,0,0,"-"]},"google.resumable_media.common":{InvalidResponse:[0,1,1,""],MAX_CUMULATIVE_RETRY:[0,4,1,""],MAX_SLEEP:[0,4,1,""],PERMANENT_REDIRECT:[0,4,1,""],RetryStrategy:[0,5,1,""],TOO_MANY_REQUESTS:[0,4,1,""],UPLOAD_CHUNK_SIZE:[0,4,1,""]},"google.resumable_media.common.InvalidResponse":{args:[0,2,1,""],response:[0,2,1,""],with_traceback:[0,3,1,""]},"google.resumable_media.common.RetryStrategy":{max_cumulative_retry:[0,2,1,""],max_retries:[0,2,1,""],max_sleep:[0,2,1,""],retry_allowed:[0,3,1,""]},"google.resumable_media.requests":{download:[2,0,0,"-"],upload:[3,0,0,"-"]},"google.resumable_media.requests.download":{ChunkedDownload:[2,5,1,""],Download:[2,5,1,""]},"google.resumable_media.requests.download.ChunkedDownload":{bytes_downloaded:[2,2,1,""],chunk_size:[2,2,1,""],consume_next_chunk:[2,3,1,""],end:[2,2,1,""],finished:[2,2,1,""],invalid:[2,2,1,""],media_url:[2,2,1,""],start:[2,2,1,""],total_bytes:[2,2,1,""]},"google.resumable_media.requests.download.Download":{consume:[2,3,1,""],end:[2,2,1,""],finished:[2,2,1,""],media_url:[2,2,1,""],start:[2,2,1,""]},"google.resumable_media.requests.upload":{MultipartUpload:[3,5,1,""],ResumableUpload:[3,5,1,""],SimpleUpload:[3,5,1,""]},"google.resumable_media.requests.upload.MultipartUpload":{finished:[3,2,1,""],transmit:[3,3,1,""],upload_url:[3,2,1,""]},"google.resumable_media.requests.upload.ResumableUpload":{bytes_uploaded:[3,2,1,""],chunk_size:[3,2,1,""],finished:[3,2,1,""],initiate:[3,3,1,""],invalid:[3,2,1,""],recover:[3,3,1,""],resumable_url:[3,2,1,""],total_bytes:[3,2,1,""],transmit_next_chunk:[3,3,1,""],upload_url:[3,2,1,""]},"google.resumable_media.requests.upload.SimpleUpload":{finished:[3,2,1,""],transmit:[3,3,1,""],upload_url:[3,2,1,""]},google:{resumable_media:[4,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","data","Python data"],"5":["py","class","Python class"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:method","4":"py:data","5":"py:class"},terms:{"1gb":1,"1mb":1,"3mb":3,"4pb4caq":1,"50mb":1,"byte":[1,2,3],"case":[1,3],"class":[0,1,2,3],"default":[0,1,2,3],"final":[1,3],"float":0,"import":[1,3],"int":[0,1,2,3],"public":4,"return":[0,2,3],"short":1,"true":[1,3],"try":[1,3],For:[0,1],GCS:1,QPS:1,The:[0,1,2,3],These:3,Used:0,Using:3,__traceback__:0,_download:2,_helper:[2,3],_upload:3,abcdef189xy_super_seri:1,abl:[1,2],about:1,accept:1,access:1,accumul:0,achiev:1,acl:3,actual:1,added:[0,3],addit:1,after:[0,1],again:3,all:[0,1,3],allow:[0,1],along:1,alreadi:[0,2],also:[1,3],alt:1,among:1,amount:0,ani:[1,3],anoth:0,api:[2,3],applic:3,arg:0,argument:[0,1,3],assign:3,associ:1,assum:[1,3],attach:3,attempt:0,auth:1,authent:[1,2,3],authorizedsess:1,avoid:1,base:[0,2,3],basic:1,been:[0,1,2,3],befor:[1,3],begin:[1,2,3],being:[1,3],best:1,between:0,big:1,bite:3,blob_nam:1,bool:[0,2,3],both:[0,1,3],bucket:[1,3],bytes_download:[1,2],bytes_upload:[1,3],bytesio:[1,3],call:[2,3],caller:[0,1,3],can:[0,1,2,3],cannot:[1,3],cap:0,caught_exc:[1,3],caus:[0,1],chang:1,check:[0,1],chosen:0,chunk:[0,2,3],chunk_siz:[1,2,3],chunkeddownload:[1,2],client:[0,1,3],cloud:1,code:[0,1,3],color:1,com:[1,3],come:[0,1],common:4,complet:[0,1,2,3],concaten:2,conclud:3,configur:0,connect:1,consid:[1,3],constant:0,construct:[1,3],consum:[1,2],consume_next_chunk:[1,2],contain:[2,3,4],content:[1,3],content_typ:[1,3],contenttyp:1,contrast:1,correct:0,cours:1,creat:[1,3],credenti:1,cumul:0,current:[2,3],custom:0,data:[1,2,3],depend:0,determin:[1,3],deviat:1,devstorag:1,directli:3,disk:1,doesn:1,done:[1,3],download:[0,4],drop:1,due:3,dure:0,dynam:3,each:[1,2,3,4],either:0,enclos:1,encrypt:[2,3],end:[1,2],error:[0,1,3],essenti:1,even:[1,3],everi:3,exce:0,except:[0,1,3],expect:[1,2,3],explicitli:3,extra:[1,2,3],fail:[0,1,2,3],failur:[0,1,3],fals:[1,3],fed:3,fewer:3,file:[1,2,3],filenam:3,finish:[1,2,3],first:[1,2],fit:1,flag:[2,3],fly:1,format:[1,3],from:[1,2,3],fulfil:3,gener:[0,1,4],get:3,getsiz:3,googleapi:[1,3],grurpl:1,gupload:1,happen:3,has:[0,1,2,3,4],have:[1,2,3],header:[1,2,3],helper:[0,2],here:3,how:1,howev:[1,2,3],http:[0,1,2,3],httpstatu:[1,3],ignor:3,imag:3,implement:1,implicitli:3,includ:0,indic:[0,2,3],inform:0,initi:[1,3],input:1,instanc:1,intend:1,interfac:[1,4],internet:1,invalid:[2,3],invalidrespons:[0,1,3],itself:3,jpeg:3,json:1,json_respons:1,kei:2,know:1,known:[1,3],larg:1,last:2,latenc:1,len:[1,3],length:1,let:1,librari:[1,4],like:[2,3],limit:[0,1],list:3,live:1,locat:1,log:3,m0xlesx9:1,mai:1,major:4,make:[1,2,3],manag:2,map:[2,3],max_cumulative_retri:0,max_retri:0,max_sleep:0,maximum:0,md5hash:1,media:[0,1,2,3,4],media_url:[1,2],memori:1,metadata:[1,3],method:3,might:3,minut:0,modul:4,more:[0,1,3],most:0,much:1,multi:1,multipart:3,multipartupload:[1,3],multipl:[0,1,3],must:[0,1,3],name:[1,3],nearest:0,neg:2,neither:[0,2],next:[2,3],none:[0,1,2,3],nor:2,num_retri:0,number:[0,2,3],object:[0,1,2,3],obtain:1,occur:[0,2,3],onc:1,one:[0,1,3],onli:[1,3],open:3,option:[0,1,2,3],other:1,out:1,over:1,packag:[1,4],parallel:1,paramet:[0,2,3],part:1,pass:[0,3],path:3,payload:3,perman:0,permanent_redirect:0,pip:4,plain:[1,3],poor:1,popul:3,portion:1,posit:[0,1],power:0,practic:1,preserv:3,process:0,produc:3,progress:[1,3],provid:[0,2],purpos:4,python:0,rais:[0,1,2,3],rang:[1,2],rare:1,rate:0,rather:1,raw:1,read:[1,3],read_onli:1,readi:3,recov:3,redirect:0,reduc:1,request:[0,4],requestsmixin:[2,3],requir:[1,3],resouc:1,resourc:[1,2,3],respons:[0,1,2,3],response0:1,response1:1,response2:1,resum:[0,3,4],resumable_url:[1,3],resumableupload:[1,3],retri:0,retriev:2,retry_allow:0,retrystrategi:0,rfc:0,ro_scop:1,same:1,scope:1,second:1,see:[0,1],seek:1,self:0,send:[1,3],sens:1,sent:[1,2,3],seri:1,server:[1,3],servic:0,session:[1,2,3],set:0,share:0,should:[1,2,3],similar:1,simpl:3,simplest:1,simpleupload:[1,3],sinc:[0,3],singl:[1,3],size:[1,3],sleep:0,slice:2,small:[1,3],smdii:1,some:[0,1,4],specif:4,specifi:[0,1,2,3],speed:1,start:[1,2],state:[0,1,2,3],statu:[0,1,3],storag:[1,3],str:[2,3],stream:[1,2,3],stream_fin:3,strictli:3,sub:1,support:[1,2,3],sure:3,tailor:4,take:1,task:1,tell:3,text:[1,3],than:[1,3],thei:[0,2],thi:[0,1,2,3,4],thing:1,those:3,three:1,thrown:3,time:[0,3],too:1,too_many_request:0,total:[0,1,2,3],total_byt:[1,2,3],total_sleep:0,tr_request:1,traffic:1,transmit:[1,3],transmit_next_chunk:[1,3],transport:[2,3,4],tupl:0,two:[0,1],type:[0,2,3],typic:[0,2],unknown:[1,3],until:[1,3],upgrad:4,upload:[0,4],upload_chunk_s:[0,3],upload_id:1,upload_url:[1,3],uploadid:1,uploadtyp:[1,3],url:[1,2,3],url_templ:[1,3],usag:[1,2],use:[1,3],used:[0,1,2,3],useful:0,user:3,uses:[1,4],using:1,util:[0,1,4],valid:3,valu:3,valueerror:[0,2,3],veri:1,verifi:3,via:3,well:1,when:[0,1,3],where:[1,3],which:[0,2,3],with_traceback:0,without:1,won:3,would:1,write:[1,2],written:[1,2,3],www:[1,3],your:3},titles:["google.resumable_media.common module","google.resumable_media.requests","google.resumable_media.requests.download module","google.resumable_media.requests.upload module","google.resumable_media"],titleterms:{author:1,chunk:1,common:0,download:[1,2],googl:[0,1,2,3,4],instal:4,modul:[0,2,3],multipart:1,request:[1,2,3],resum:1,resumable_media:[0,1,2,3,4],simpl:1,subpackag:4,transport:1,upload:[1,3]}}) \ No newline at end of file +Search.setIndex({docnames:["google.resumable_media.common","google.resumable_media.requests","google.resumable_media.requests.download","google.resumable_media.requests.upload","index"],envversion:53,filenames:["google.resumable_media.common.rst","google.resumable_media.requests.rst","google.resumable_media.requests.download.rst","google.resumable_media.requests.upload.rst","index.rst"],objects:{"google.resumable_media":{common:[0,0,0,"-"],requests:[1,0,0,"-"]},"google.resumable_media.common":{InvalidResponse:[0,1,1,""],MAX_CUMULATIVE_RETRY:[0,4,1,""],MAX_SLEEP:[0,4,1,""],PERMANENT_REDIRECT:[0,4,1,""],RetryStrategy:[0,5,1,""],TOO_MANY_REQUESTS:[0,4,1,""],UPLOAD_CHUNK_SIZE:[0,4,1,""]},"google.resumable_media.common.InvalidResponse":{args:[0,2,1,""],response:[0,2,1,""],with_traceback:[0,3,1,""]},"google.resumable_media.common.RetryStrategy":{max_cumulative_retry:[0,2,1,""],max_retries:[0,2,1,""],max_sleep:[0,2,1,""],retry_allowed:[0,3,1,""]},"google.resumable_media.requests":{download:[2,0,0,"-"],upload:[3,0,0,"-"]},"google.resumable_media.requests.download":{ChunkedDownload:[2,5,1,""],Download:[2,5,1,""]},"google.resumable_media.requests.download.ChunkedDownload":{bytes_downloaded:[2,2,1,""],chunk_size:[2,2,1,""],consume_next_chunk:[2,3,1,""],end:[2,2,1,""],finished:[2,2,1,""],invalid:[2,2,1,""],media_url:[2,2,1,""],start:[2,2,1,""],total_bytes:[2,2,1,""]},"google.resumable_media.requests.download.Download":{consume:[2,3,1,""],end:[2,2,1,""],finished:[2,2,1,""],media_url:[2,2,1,""],start:[2,2,1,""]},"google.resumable_media.requests.upload":{MultipartUpload:[3,5,1,""],ResumableUpload:[3,5,1,""],SimpleUpload:[3,5,1,""]},"google.resumable_media.requests.upload.MultipartUpload":{finished:[3,2,1,""],transmit:[3,3,1,""],upload_url:[3,2,1,""]},"google.resumable_media.requests.upload.ResumableUpload":{bytes_uploaded:[3,2,1,""],chunk_size:[3,2,1,""],finished:[3,2,1,""],initiate:[3,3,1,""],invalid:[3,2,1,""],recover:[3,3,1,""],resumable_url:[3,2,1,""],total_bytes:[3,2,1,""],transmit_next_chunk:[3,3,1,""],upload_url:[3,2,1,""]},"google.resumable_media.requests.upload.SimpleUpload":{finished:[3,2,1,""],transmit:[3,3,1,""],upload_url:[3,2,1,""]},google:{resumable_media:[4,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","data","Python data"],"5":["py","class","Python class"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:method","4":"py:data","5":"py:class"},terms:{"1gb":1,"1mb":1,"3mb":3,"4pb4caq":1,"50mb":1,"byte":[1,2,3],"case":[1,3],"class":[0,1,2,3],"default":[0,1,2,3],"final":[1,3],"float":0,"import":[1,3],"int":[0,1,2,3],"public":4,"return":[0,2,3],"short":1,"true":[1,3],"try":[1,3],For:[0,1],GCS:1,QPS:1,The:[0,1,2,3],These:3,Used:0,Using:3,__traceback__:0,_download:2,_helper:[2,3],_upload:3,abcdef189xy_super_seri:1,abl:[1,2],about:1,accept:1,access:1,accumul:0,achiev:1,acl:3,actual:1,added:[0,3],addit:1,after:[0,1],again:3,all:[0,1,3],allow:[0,1],along:1,alreadi:[0,2],also:[1,3],alt:1,among:1,amount:0,ani:[1,3],anoth:0,api:[2,3],applic:3,arg:0,argument:[0,1,3],assign:3,associ:1,assum:[1,3],attach:[2,3],attempt:0,auth:1,authent:[1,2,3],authorizedsess:1,avoid:1,base:[0,2,3],basic:1,been:[0,1,2,3],befor:[1,3],begin:[1,2,3],being:[1,3],best:1,between:0,big:1,bite:3,blob_nam:1,bool:[0,2,3],both:[0,1,3],bucket:[1,3],bytes_download:[1,2],bytes_upload:[1,3],bytesio:[1,3],call:[2,3],caller:[0,1,3],can:[0,1,2,3],cannot:[1,3],cap:0,caught_exc:[1,3],caus:[0,1],chang:1,check:[0,1],chosen:0,chunk:[0,2,3],chunk_siz:[1,2,3],chunkeddownload:[1,2],client:[0,1,3],cloud:1,code:[0,1,3],color:1,com:[1,3],come:[0,1],common:4,complet:[0,1,2,3],concaten:2,conclud:3,configur:0,connect:1,consid:[1,3],constant:0,construct:[1,3],consum:[1,2],consume_next_chunk:[1,2],contain:[2,3,4],content:[1,3],content_typ:[1,3],contenttyp:1,contrast:1,correct:0,cours:1,creat:[1,3],credenti:1,cumul:0,current:[2,3],custom:0,data:[1,2,3],depend:0,determin:[1,3],deviat:1,devstorag:1,directli:3,disk:1,doesn:1,done:[1,3],download:[0,4],drop:1,due:3,dure:0,dynam:3,each:[1,2,3,4],either:0,enclos:1,encrypt:[2,3],end:[1,2],error:[0,1,3],essenti:1,even:[1,3],everi:3,exce:0,except:[0,1,3],expect:[1,2,3],explicitli:3,extra:[1,2,3],fail:[0,1,2,3],failur:[0,1,3],fals:[1,3],fed:3,fewer:3,file:[1,2,3],filenam:3,finish:[1,2,3],first:[1,2],fit:1,flag:[2,3],fly:1,format:[1,3],from:[1,2,3],fulfil:3,gener:[0,1,4],get:3,getsiz:3,googleapi:[1,3],grurpl:1,gupload:1,happen:3,has:[0,1,2,3,4],have:[1,2,3],header:[1,2,3],helper:[0,2],here:3,how:1,howev:[1,2,3],http:[0,1,2,3],httpstatu:[1,3],ignor:3,imag:3,implement:1,implicitli:3,includ:0,indic:[0,2,3],inform:0,initi:[1,3],input:1,instanc:1,intend:1,interfac:[1,4],internet:1,invalid:[2,3],invalidrespons:[0,1,3],itself:3,jpeg:3,json:1,json_respons:1,kei:2,know:1,known:[1,3],larg:1,last:2,latenc:1,len:[1,3],length:1,let:1,librari:[1,4],like:[2,3],limit:[0,1],list:3,live:1,locat:1,log:3,m0xlesx9:1,mai:1,major:4,make:[1,2,3],manag:2,map:[2,3],max_cumulative_retri:0,max_retri:0,max_sleep:0,maximum:0,md5hash:1,media:[0,1,2,3,4],media_url:[1,2],memori:1,metadata:[1,3],method:3,might:3,minut:0,modul:4,more:[0,1,3],most:0,much:1,multi:1,multipart:3,multipartupload:[1,3],multipl:[0,1,3],must:[0,1,3],name:[1,3],nearest:0,neg:2,neither:[0,2],next:[2,3],none:[0,1,2,3],nor:2,num_retri:0,number:[0,2,3],object:[0,1,2,3],obtain:1,occur:[0,2,3],onc:1,one:[0,1,3],onli:[1,3],open:3,option:[0,1,2,3],other:1,out:1,over:1,packag:[1,4],parallel:1,paramet:[0,2,3],part:1,pass:[0,3],path:3,payload:3,perman:0,permanent_redirect:0,pip:4,plain:[1,3],poor:1,popul:3,portion:1,posit:[0,1],power:0,practic:1,preserv:3,process:0,produc:3,progress:[1,3],provid:[0,2],purpos:4,python:0,rais:[0,1,2,3],rang:[1,2],rare:1,rate:0,rather:1,raw:1,read:[1,3],read_onli:1,readi:3,recov:3,redirect:0,reduc:1,request:[0,4],requestsmixin:[2,3],requir:[1,3],resouc:1,resourc:[1,2,3],respons:[0,1,2,3],response0:1,response1:1,response2:1,resum:[0,3,4],resumable_url:[1,3],resumableupload:[1,3],retri:0,retriev:2,retry_allow:0,retrystrategi:0,rfc:0,ro_scop:1,same:1,scope:1,second:1,see:[0,1],seek:1,self:0,send:[1,3],sens:1,sent:[1,2,3],seri:1,server:[1,3],servic:0,session:[1,2,3],set:0,share:0,should:[1,2,3],similar:1,simpl:3,simplest:1,simpleupload:[1,3],sinc:[0,3],singl:[1,3],size:[1,3],sleep:0,slice:2,small:[1,3],smdii:1,some:[0,1,4],specif:4,specifi:[0,1,2,3],speed:1,start:[1,2],state:[0,1,2,3],statu:[0,1,3],storag:[1,3],str:[2,3],stream:[1,2,3],stream_fin:3,strictli:3,sub:1,support:[1,2,3],sure:3,tailor:4,take:1,task:1,tell:3,text:[1,3],than:[1,3],thei:[0,2],thi:[0,1,2,3,4],thing:1,those:3,three:1,thrown:3,time:[0,3],too:1,too_many_request:0,total:[0,1,2,3],total_byt:[1,2,3],total_sleep:0,tr_request:1,traffic:1,transmit:[1,3],transmit_next_chunk:[1,3],transport:[2,3,4],tupl:0,two:[0,1],type:[0,2,3],typic:[0,2],unknown:[1,3],until:[1,3],upgrad:4,upload:[0,4],upload_chunk_s:[0,3],upload_id:1,upload_url:[1,3],uploadid:1,uploadtyp:[1,3],url:[1,2,3],url_templ:[1,3],usag:[1,2],use:[1,3],used:[0,1,2,3],useful:0,user:3,uses:[1,4],using:1,util:[0,1,4],valid:3,valu:3,valueerror:[0,2,3],veri:1,verifi:3,via:3,well:1,when:[0,1,3],where:[1,3],which:[0,2,3],with_traceback:0,without:1,won:3,would:1,write:[1,2],written:[1,2,3],www:[1,3],your:3},titles:["google.resumable_media.common module","google.resumable_media.requests","google.resumable_media.requests.download module","google.resumable_media.requests.upload module","google.resumable_media"],titleterms:{author:1,chunk:1,common:0,download:[1,2],googl:[0,1,2,3,4],instal:4,modul:[0,2,3],multipart:1,request:[1,2,3],resum:1,resumable_media:[0,1,2,3,4],simpl:1,subpackag:4,transport:1,upload:[1,3]}}) \ No newline at end of file diff --git a/google/resumable_media/requests/_helpers.py b/google/resumable_media/requests/_helpers.py index 30d0870b..9ece62a3 100644 --- a/google/resumable_media/requests/_helpers.py +++ b/google/resumable_media/requests/_helpers.py @@ -73,7 +73,7 @@ def _get_body(response): def http_request(transport, method, url, data=None, headers=None, - retry_strategy=_DEFAULT_RETRY_STRATEGY): + retry_strategy=_DEFAULT_RETRY_STRATEGY, **transport_kwargs): """Make an HTTP request. Args: @@ -88,11 +88,14 @@ def http_request(transport, method, url, data=None, headers=None, may also add additional headers). retry_strategy (~google.resumable_media.common.RetryStrategy): The strategy to use if the request fails and must be retried. + transport_kwargs (Dict[str, str]): Extra keyword arguments to be + passed along to ``transport.request``. Returns: ~requests.Response: The return value of ``transport.request()``. """ func = functools.partial( - transport.request, method, url, data=data, headers=headers) + transport.request, method, url, data=data, headers=headers, + **transport_kwargs) return _helpers.wait_and_retry( func, RequestsMixin._get_status_code, retry_strategy) diff --git a/google/resumable_media/requests/download.py b/google/resumable_media/requests/download.py index 26935cfc..cda6f593 100644 --- a/google/resumable_media/requests/download.py +++ b/google/resumable_media/requests/download.py @@ -19,6 +19,9 @@ from google.resumable_media.requests import _helpers +_SINGLE_GET_CHUNK_SIZE = 8192 + + class Download(_helpers.RequestsMixin, _download.Download): """Helper to manage downloading a resource from a Google API. @@ -45,9 +48,29 @@ class Download(_helpers.RequestsMixin, _download.Download): end (Optional[int]): The last byte in a range to be downloaded. """ + def _write_to_stream(self, response): + """Write response body to a write-able stream. + + .. note: + + This method assumes that the ``_stream`` attribute is set on the + current download. + + Args: + response (~requests.Response): The HTTP response object. + """ + with response: + body_iter = response.iter_content( + chunk_size=_SINGLE_GET_CHUNK_SIZE, decode_unicode=False) + for chunk in body_iter: + self._stream.write(chunk) + def consume(self, transport): """Consume the resource to be downloaded. + If a ``stream`` is attached to this download, then the downloaded + resource will be written to the stream. + Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. @@ -61,9 +84,20 @@ def consume(self, transport): """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. + request_kwargs = { + u'data': payload, + u'headers': headers, + u'retry_strategy': self._retry_strategy, + } + if self._stream is not None: + request_kwargs[u'stream'] = True + result = _helpers.http_request( - transport, method, url, data=payload, headers=headers, - retry_strategy=self._retry_strategy) + transport, method, url, **request_kwargs) + + if self._stream is not None: + self._write_to_stream(result) + self._process_response(result) return result diff --git a/tests/system/requests/test_download.py b/tests/system/requests/test_download.py index db042532..d24166e5 100644 --- a/tests/system/requests/test_download.py +++ b/tests/system/requests/test_download.py @@ -34,6 +34,8 @@ PLAIN_TEXT = u'text/plain' ENCRYPTED_ERR = ( b'The target object is encrypted by a customer-supplied encryption key.') +NO_BODY_ERR = ( + u'The content for this response was already consumed') @pytest.fixture(scope=u'module') @@ -99,6 +101,29 @@ def test_download_full(add_files, authorized_transport): check_tombstoned(download, authorized_transport) +def test_download_to_stream(add_files, authorized_transport): + for img_file in IMG_FILES: + with open(img_file, u'rb') as file_obj: + actual_contents = file_obj.read() + + blob_name = os.path.basename(img_file) + + # Create the actual download object. + media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) + stream = io.BytesIO() + download = resumable_requests.Download(media_url, stream=stream) + # Consume the resource. + response = download.consume(authorized_transport) + assert response.status_code == http_client.OK + with pytest.raises(RuntimeError) as exc_info: + getattr(response, u'content') + assert exc_info.value.args == (NO_BODY_ERR,) + assert response._content is False + assert response._content_consumed is True + assert stream.getvalue() == actual_contents + check_tombstoned(download, authorized_transport) + + @pytest.fixture(scope=u'module') def secret_file(authorized_transport): blob_name = u'super-seekrit.txt' diff --git a/tests/unit/requests/test__helpers.py b/tests/unit/requests/test__helpers.py index 31a6a124..2a202bb4 100644 --- a/tests/unit/requests/test__helpers.py +++ b/tests/unit/requests/test__helpers.py @@ -43,11 +43,13 @@ def test_http_request(): data = mock.sentinel.data headers = {u'one': u'fish', u'blue': u'fish'} ret_val = _helpers.http_request( - transport, method, url, data=data, headers=headers) + transport, method, url, data=data, headers=headers, + extra1=b'work', extra2=125.5) assert ret_val is responses[0] transport.request.assert_called_once_with( - method, url, data=data, headers=headers) + method, url, data=data, headers=headers, + extra1=b'work', extra2=125.5) def _make_response(status_code): diff --git a/tests/unit/requests/test_download.py b/tests/unit/requests/test_download.py index 5c53148e..1e16b3af 100644 --- a/tests/unit/requests/test_download.py +++ b/tests/unit/requests/test_download.py @@ -28,24 +28,67 @@ class TestDownload(object): - def _consume_helper(self, end=65536, headers=None): - download = download_mod.Download(EXAMPLE_URL, end=end, headers=headers) + def test__write_to_stream(self): + stream = io.BytesIO() + download = download_mod.Download(EXAMPLE_URL, stream=stream) + + chunk1 = b'right now, ' + chunk2 = b'but a little later' + response = _mock_response(chunks=[chunk1, chunk2]) + + ret_val = download._write_to_stream(response) + assert ret_val is None + + assert stream.getvalue() == chunk1 + chunk2 + + # Check mocks. + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) + + def _consume_helper(self, stream=None, end=65536, headers=None, chunks=()): + download = download_mod.Download( + EXAMPLE_URL, stream=stream, end=end, headers=headers) transport = mock.Mock(spec=[u'request']) - transport.request.return_value = mock.Mock( - status_code=int(http_client.OK), spec=[u'status_code']) + transport.request.return_value = _mock_response(chunks=chunks) assert not download.finished ret_val = download.consume(transport) assert ret_val is transport.request.return_value + + called_kwargs = {u'data': None, u'headers': download._headers} + if chunks: + assert stream is not None + called_kwargs[u'stream'] = True transport.request.assert_called_once_with( - u'GET', EXAMPLE_URL, data=None, headers=download._headers) + u'GET', EXAMPLE_URL, **called_kwargs) + range_bytes = u'bytes={:d}-{:d}'.format(0, end) assert download._headers[u'range'] == range_bytes assert download.finished + return transport + def test_consume(self): self._consume_helper() + def test_consume_with_stream(self): + stream = io.BytesIO() + chunks = (b'up down ', b'charlie ', b'brown') + transport = self._consume_helper(stream=stream, chunks=chunks) + + assert stream.getvalue() == b''.join(chunks) + + # Check mocks. + response = transport.request.return_value + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) + def test_consume_with_headers(self): headers = {} # Empty headers end = 16383 @@ -121,3 +164,21 @@ def test_consume_next_chunk(self): assert not download.finished assert download.bytes_downloaded == chunk_size assert download.total_bytes == total_bytes + + +def _mock_response(status_code=http_client.OK, chunks=()): + if chunks: + response = mock.MagicMock( + status_code=int(status_code), + spec=[u'__enter__', u'__exit__', u'iter_content', u'status_code'], + ) + # i.e. context manager returns ``self``. + response.__enter__.return_value = response + response.__exit__.return_value = None + response.iter_content.return_value = iter(chunks) + return response + else: + return mock.Mock( + status_code=int(status_code), + spec=[u'status_code'], + )