Skip to content

Commit

Permalink
fix azureblob tests: use json instead of numpy (spotify#3032)
Browse files Browse the repository at this point in the history
Numpy is not installed in the current setup

It fixes these errors:

```
  File "/home/travis/build/spotify/luigi/test/contrib/azureblob_test.py", line 138, in run

    import numpy

ModuleNotFoundError: No module named 'numpy'
```
https://travis-ci.org/github/spotify/luigi/jobs/754449469

The test silently failed, asserts weren't covered
Added an assert checking the output of FinalTask
  • Loading branch information
vbarbaresi authored and hirosassa committed Feb 10, 2021
1 parent 59eaacc commit 27fcb45
Showing 1 changed file with 11 additions and 11 deletions.
22 changes: 11 additions & 11 deletions test/contrib/azureblob_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import os
import unittest
import json

from nose.plugins.attrib import attr

Expand Down Expand Up @@ -129,31 +130,27 @@ def run(self):
op.write("Greed, for lack of a better word, is good.\n")


class AzureNumpyDumpTask(luigi.Task):
class AzureJsonDumpTask(luigi.Task):
def output(self):
return AzureBlobTarget("luigi-test", "stats.npy", client, format=luigi.format.Nop)
return AzureBlobTarget("luigi-test", "stats.json", client)

def run(self):
with self.output().open("w") as op:
import numpy
numpy.save(op, numpy.array([1, 2, 3]))
json.dump([1, 2, 3], op)


class FinalTask(luigi.Task):
def requires(self):
return {"movie": self.clone(MovieScriptTask), "np": self.clone(AzureNumpyDumpTask)}
return {"movie": self.clone(MovieScriptTask), "np": self.clone(AzureJsonDumpTask)}

def run(self):
with self.input()["movie"].open('r') as movie, self.input()["np"].open('r') as np, self.output().open('w') as output:
movie_lines = movie.read()
assert "Toto, I've got a feeling" in movie_lines
output.write(movie_lines)

import numpy
data = numpy.load(np)
assert data[0] == 1
assert data[1] == 2
assert data[2] == 3
data = json.load(np)
assert data == [1, 2, 3]
output.write(data.__str__())

def output(self):
Expand All @@ -169,4 +166,7 @@ def tearDown(self):
pass

def test_AzureBlobTarget(self):
luigi.build([FinalTask()], local_scheduler=True, log_level='NOTSET')
final_task = FinalTask()
luigi.build([final_task], local_scheduler=True, log_level='NOTSET')
output = final_task.output().open("r").read()
assert "Toto" in output

0 comments on commit 27fcb45

Please sign in to comment.