do not use deprecated luigi.File

prepare for: spotify/luigi#1915
miku · Nov 7, 2016 · 7cd1bda · 7cd1bda · Tarrasch · Nov 8, 2016
1 parent 79df8dc
commit 7cd1bda
Show file tree

Hide file tree

Showing 17 changed files with 92 additions and 92 deletions.
diff --git a/siskin/common.py b/siskin/common.py
@@ -147,7 +147,7 @@ def run(self):
                           username=pipes.quote(self.username),
                           password=pipes.quote(self.password),
                           filepath=pipes.quote(self.filepath))
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(digest=True, ext=None))
diff --git a/siskin/sources/amsl.py b/siskin/sources/amsl.py
@@ -71,7 +71,7 @@ def run(self):
 
         link = '%s/%s/list?do=%s' % (self.config.get('amsl', 'base').rstrip('/'), realm, name)
         output = shellout("""curl --fail "{link}" > {output} """, link=link)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(digest=True))
@@ -244,7 +244,7 @@ def run(self):
                     # at least the file is not a zip.
                     output = shellout("cat {input} >> {output}", input=downloaded, output=stopover)
 
-        luigi.File(stopover).move(self.output().path)
+        luigi.LocalTarget(stopover).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path())
@@ -279,7 +279,7 @@ def run(self):
         shellout("cut -f 2 {input} | grep -oE '[0-9]{{4,4}}-[xX0-9]{{4,4}}' >> {output}", input=output, output=stopover)
         shellout("cut -f 3 {input} | grep -oE '[0-9]{{4,4}}-[xX0-9]{{4,4}}' >> {output}", input=output, output=stopover)
         output = shellout("sort -u {input} > {output}", input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path())

diff --git a/siskin/sources/arxiv.py b/siskin/sources/arxiv.py
@@ -81,7 +81,7 @@ def run(self):
                  prefix=self.prefix, url=self.url, dir=self.config.get('core', 'metha-dir'))
         output = shellout("METHA_DIR={dir} metha-cat -format {prefix} {url} | pigz -c > {output}",
                           prefix=self.prefix, url=self.url, dir=self.config.get('core', 'metha-dir'))
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="xml.gz"))
@@ -100,7 +100,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout("span-import -i oai <(unpigz -c {input}) | pigz -c > {output}", input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="xml.gz"))
@@ -117,7 +117,7 @@ def requires(self):
 
     def run(self):
         output = shellout("span-export -o {format} <(unpigz -c {input}) | pigz -c > {output}", format=self.format, input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         extensions = {

diff --git a/siskin/sources/base.py b/siskin/sources/base.py
@@ -67,7 +67,7 @@ def run(self):
                  set=self.set, prefix=self.prefix, dir=self.config.get('core', 'metha-dir'))
         output = shellout("METHA_DIR={dir} metha-cat -set '{set}' -format {prefix} http://oai.base-search.net/oai | pigz -c > {output}",
                           set=self.set, prefix=self.prefix, dir=self.config.get('core', 'metha-dir'))
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="xml.gz", digest=True))
diff --git a/siskin/sources/crossref.py b/siskin/sources/crossref.py
@@ -172,7 +172,7 @@ def requires(self):
 
     def run(self):
         output = shellout("jq -c -r '.message.items[]?' {input} | pigz -c > {output}", input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'), format=Gzip)
@@ -216,7 +216,7 @@ def requires(self):
 
     def run(self):
         output = shellout(r"""jq -r '.DOI?' <(unpigz -c {input}) | awk '{{print NR"\t{input}\t"$0 }}' | pigz -c > {output}""", input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='filelist.gz'))
@@ -254,7 +254,7 @@ def run(self):
         _, stopover = tempfile.mkstemp(prefix='siskin-')
         for target in self.input():
             shellout("cat {input} >> {output}", input=target.path, output=stopover)
-        luigi.File(stopover).move(self.output().path)
+        luigi.LocalTarget(stopover).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='filelist.gz'))
@@ -277,7 +277,7 @@ def run(self):
             TMPDIR={tmpdir} LC_ALL=C sort -S25% -u -k3,3 |
             pigz -c > {output}""", tmpdir=self.config.get('core', 'tempdir'),
             input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='filelist.gz'), format=Gzip)
@@ -341,7 +341,7 @@ def run(self):
         output = shellout("""
             TMPDIR={tmpdir} LC_ALL=C sort -S50% -k2,2 -k1,1n <(TMPDIR={tmpdir} unpigz -c {input}) | cut -f 1-2 | pigz -c > {output}""",
             tmpdir=self.config.get('core', 'tempdir'), input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='filelist.gz'), format=Gzip)
@@ -375,7 +375,7 @@ def run(self):
                 lineno, filename = line.strip().split('\t')
                 if previous_filename and previous_filename != filename:
                     _, tmp = tempfile.mkstemp(prefix='siskin-')
-                    with luigi.File(tmp, format=TSV).open('w') as handle:
+                    with luigi.LocalTarget(tmp, format=TSV).open('w') as handle:
                         for ln in linenumbers:
                             handle.write_tsv(ln)
 
@@ -396,7 +396,7 @@ def run(self):
                 linenumbers.append(lineno)
                 previous_filename = filename
 
-        luigi.File(stopover).move(self.output().path)
+        luigi.LocalTarget(stopover).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'))
@@ -416,7 +416,7 @@ def requires(self):
     def run(self):
         output = shellout("span-import -i crossref <(unpigz -c {input}) | pigz -c > {output}",
                           input=self.input().get('file').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'))
@@ -438,7 +438,7 @@ def run(self):
         output = shellout("span-tag -c {config} <(unpigz -c {input}) | pigz -c > {output}",
                           config=self.input().get('config').path, input=self.input().get('file').path)
         output = shellout("span-export -o {format} <(unpigz -c {input}) | pigz -c > {output}", format=self.format, input=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         extensions = {
@@ -462,7 +462,7 @@ def requires(self):
     def run(self):
         output = shellout("""jq -r '.["finc.mega_collection"]?' <(unpigz -c {input}) | LC_ALL=C sort -S35% -u > {output}""",
                           input=self.input().get('input').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -520,7 +520,7 @@ def run(self):
                  input=output, output=stopover)
         os.remove(output)
         output = shellout("""sort -S50% -u {input} > {output} """, input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -539,7 +539,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout("jq -r '.ISSN[]?' <(unpigz -c {input}) 2> /dev/null > {output}", input=self.input().get('input').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -557,7 +557,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout("sort -u {input} > {output}", input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -579,7 +579,7 @@ def run(self):
         output = shellout("""jq -r '[.doi?, .["rft.issn"][]?, .["rft.eissn"][]?] | @csv' {input} | LC_ALL=C sort -S50% > {output} """,
                           input=temp, output=stopover)
         os.remove(temp)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='csv'))
@@ -723,7 +723,7 @@ def requires(self):
 
     def run(self):
         output = shellout("hurrly -w 64 < {input} | pigz > {output}", input=self.input().get('input').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='tsv.gz'))
@@ -748,7 +748,7 @@ def run(self):
         shellout("""LC_ALL=C zgrep -v "^200" {input} >> {output}""",
              input=self.input().path, output=stopover)
         output = shellout("sort -S50% -u {input} | cut -f4 | sed s@http://doi.org/api/handles/@@g > {output}", input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path())
diff --git a/siskin/sources/dblp.py b/siskin/sources/dblp.py
@@ -45,7 +45,7 @@ class DBLPDownload(DBLPTask):
 
     def run(self):
         output = shellout("curl --fail -L {url} > {output}", url=self.url)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='xml.gz'))
@@ -61,7 +61,7 @@ def requires(self):
     def run(self):
         output = shellout("""LC_ALL=C grep "doi.org" <(unpigz -c {input}) | LC_ALL=C sed -e 's@<ee>http://dx.doi.org/@@g' |
                              LC_ALL=C sed -e 's@</ee>@@g' | LC_ALL=C sort -S50% > {output}""", input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path())
diff --git a/siskin/sources/degruyter.py b/siskin/sources/degruyter.py
@@ -90,7 +90,7 @@ def run(self):
             for row in handle.iter_tsv(cols=('path',)):
                 shellout(""" unzip -l {input} | grep "xml$" | awk '{{print "{input}\t"$4}}' >> {output} """,
                          preserve_whitespace=True, input=row.path, output=stopover)
-        luigi.File(stopover).move(self.output().path)
+        luigi.LocalTarget(stopover).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -115,7 +115,7 @@ def run(self):
                     continue
                 shellout("unzip -p {path} \*.xml 2> /dev/null >> {output}", output=stopover, path=row.path,
                          ignoremap={1: 'OK', 9: 'skip corrupt file'})
-        luigi.File(stopover).move(self.output().path)
+        luigi.LocalTarget(stopover).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='xml'), format=TSV)
@@ -132,7 +132,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout("span-import -i degruyter {input} | pigz -c > {output}", input=self.input().get('file').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'))
@@ -155,7 +155,7 @@ def run(self):
         output = shellout("span-tag -c {config} <(unpigz -c {input}) | pigz -c > {output}",
                           config=self.input().get('config').path, input=self.input().get('file').path)
         output = shellout("span-export -o {format} <(unpigz -c {input}) | pigz -c > {output}", input=output, format=self.format)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         extensions = {
@@ -178,7 +178,7 @@ def run(self):
         shellout("""jq -r '.["rft.issn"][]?' <(unpigz -c {input}) 2> /dev/null >> {output} """, input=self.input().path, output=stopover)
         shellout("""jq -r '.["rft.eissn"][]?' <(unpigz -c {input}) 2> /dev/null >> {output} """, input=self.input().path, output=stopover)
         output = shellout("""sort -u {input} > {output} """, input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -196,7 +196,7 @@ def run(self):
         _, stopover = tempfile.mkstemp(prefix='siskin-')
         shellout("""jq -r '.doi' <(unpigz -c {input}) | grep -v "null" | grep -o "10.*" 2> /dev/null > {output} """, input=self.input().get('input').path, output=stopover)
         output = shellout("""sort -u {input} > {output} """, input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
diff --git a/siskin/sources/doaj.py b/siskin/sources/doaj.py
@@ -71,7 +71,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout('wget --retry-connrefused {url} -O {output}', url=self.url)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='csv'))
@@ -175,7 +175,7 @@ def requires(self):
     @timed
     def run(self):
         output = shellout("span-import -i doaj {input} | pigz -c > {output}", input=self.input().get('input').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'))
@@ -192,7 +192,7 @@ def requires(self):
 
     def run(self):
         output = shellout("span-export -o {format} <(unpigz -c {input}) | pigz -c > {output}", format=self.format, input=self.input().path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         extensions = {
@@ -217,7 +217,7 @@ def run(self):
         shellout("""jq -r '.["rft.issn"][]?' <(unpigz -c {input}) >> {output} """, input=self.input().get('input').path, output=stopover)
         shellout("""jq -r '.["rft.eissn"][]?' <(unpigz -c {input}) >> {output} """, input=self.input().get('input').path, output=stopover)
         output = shellout("""sort -u {input} > {output} """, input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -236,7 +236,7 @@ def requires(self):
     def run(self):
         output = shellout("""jq -r '.doi' <(unpigz -c {input}) | grep -v "null" | grep -o "10.*" 2> /dev/null | sort -u > {output} """,
                           input=self.input().get('input').path)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
diff --git a/siskin/sources/elsevierjournals.py b/siskin/sources/elsevierjournals.py
@@ -74,7 +74,7 @@ def run(self):
         _, output = tempfile.mkstemp(prefix='siskin-')
         for path in sorted(iterfiles(directory, fun=lambda p: p.endswith('.tar'))):
             shellout("span-import -i elsevier-tar {input} | pigz -c >> {output}", input=path, output=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'), format=Gzip)
@@ -119,7 +119,7 @@ def run(self):
                 if not row.path.endswith('.tar'):
                     continue
                 shellout("span-import -i elsevier-tar {input} | pigz -c >> {output}", input=row.path, output=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'), format=Gzip)
@@ -137,7 +137,7 @@ def run(self):
         _, output = tempfile.mkstemp(prefix='siskin-')
         for target in self.input():
             shellout("cat {input} >> {output}", input=target.path, output=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext='ldj.gz'), format=Gzip)
@@ -159,7 +159,7 @@ def run(self):
         output = shellout("span-tag -c {config} <(unpigz -c {input}) | pigz -c > {output}",
                           config=self.input().get('config').path, input=self.input().get('file').path)
         output = shellout("span-export -o {format} <(unpigz -c {input}) | pigz -c > {output}", format=self.format, input=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         extensions = {
@@ -187,7 +187,7 @@ def run(self):
                  input=output, output=stopover)
         os.remove(output)
         output = shellout("""sort -S50% -u {input} > {output} """, input=stopover)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)
@@ -208,7 +208,7 @@ def run(self):
         shellout("""jq -c -r '.["rft.issn"][]?' <(unpigz -c {input}) >> {output} """, input=self.input().get('input').path, output=output)
         shellout("""jq -c -r '.["rft.eissn"][]?' <(unpigz -c {input}) >> {output} """, input=self.input().get('input').path, output=output)
         output = shellout("""sort -u {input} > {output} """, input=output)
-        luigi.File(output).move(self.output().path)
+        luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(), format=TSV)