From 57e9a0626a80353b9eef590a2959b3ceee72d1a1 Mon Sep 17 00:00:00 2001 From: Yan Wang Date: Thu, 28 Sep 2023 18:24:32 -0700 Subject: [PATCH 1/2] Update the Dockerfiles --- docker/README.md | 64 +++++++++++++++++++++++++++++++++--- docker/solver/Dockerfile | 24 +++++++------- docker/webservice/Dockerfile | 27 +++++++++------ docker/webservice/config | 8 +++++ net/.gitignore | 1 + net/settings_test.py | 2 ++ 6 files changed, 101 insertions(+), 25 deletions(-) create mode 100755 docker/webservice/config create mode 100755 net/.gitignore diff --git a/docker/README.md b/docker/README.md index 9abd02596..e45bf1a47 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,65 @@ +# Astrometry.net Web Interface Dockerfiles -Docker containers for Astrometry.net +Many people visit this repository with the intention of deploying a local version of the astrometry.net web interface. +Although it's entirely feasible to manually follow the instructions in the `net`` folder, we also provide docker files to enable a quick and efficient deployment. +However, it's important to note that these docker files are primarily designed for quick trials and development. +If you're planning to use them for production, they'll require further refinement, particularly in terms of security settings. -(cd solver && docker build -t astrometrynet/solver:latest .) +This folder contains two docker files. +The first is for the solver, which provides the command line tools for astrometry.net. +The second is for the web service, which includes the Django-based web server and API server. +Here's how you can set up a docker container using these docker files. -(cd webservice && docker build -t astrometrynet/webservice:latest .) +## solver -docker run --net=host astrometrynet/webservice +In most cases, the solver requires an index to function. +A common approach is to download the pre-cooked index files. +One can use the command line provided below to download. +Please note that all the command lines referenced in this document assume that they are being executed under the repository root, such as `~/astrometry.net`, and not the current folder. +``` +mkdir -p ../astrometry_indexes +pushd ../astrometry_indexes/ \ +&& for i in 4100 4200; do \ + wget -r -l1 --no-parent -nc -nd -A ".fits" http://data.astrometry.net/$i/;\ +done +popd +``` +The index could be fairly large, and it's likely you only need part of them. +Check out [this link](http://astrometry.net/doc/readme.html#getting-index-files) to understand whether it's possible to only download and use part of all the files. +Otherwise, downloading all the files will also work. +If web service is also desired, it's a good time to config the security settings (`appsecrets`). +`appsecrets-example` is a good start point. +If it's only for a quick peek, `cp -ar net/appsecrets-example net/appsecrets` is good enough. + +Then one can build the docker image using the command line: +``` +sudo docker build -t astrometrynet/solver:latest -f docker/solver/Dockerfile . +``` +Again note the command should be executed in the repo root folder, not the current folder. + +Then use this command to log into the container to use the command lines: +``` +sudo docker run -v ~/astrometry_indexes:/usr/local/data -it astrometrynet/solver /bin/bash +``` +Here `~/astrometry_indexes` is the host folder holding the indexes downloaded from the first step. +In the `solver` container, the command line tools are available for use. +For example, `solve-field`. + +## webservice + +This container depends on the `solver` container. +First follow the steps in the previous section to build the `solver` container. +Note if you made any changes to the repo, e.g. changing the secrets in the `appsecrets`, `solver` container needs to be rebuilt for the changes to take effect. + +Then build the `webservice` container: +``` +sudo docker build -t astrometrynet/webservice:latest -f docker/webservice/Dockerfile . +``` + +For the container to function properly, we still need to map the indexes folder to it, with some port mapping: +``` +sudo docker run -p 8000:8000 -v ~/astrometry_indexes:/usr/local/data astrometrynet/webservice +``` + +The the Astrometry.net website could be accessed on the host machine at http://localhost:8000. \ No newline at end of file diff --git a/docker/solver/Dockerfile b/docker/solver/Dockerfile index d5650db3b..46431f174 100644 --- a/docker/solver/Dockerfile +++ b/docker/solver/Dockerfile @@ -34,7 +34,7 @@ RUN apt -y update && apt install -y apt-utils && \ python3-numpy \ python3-scipy \ python3-matplotlib \ - && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + && apt-get clean # Pip installs RUN for x in \ @@ -47,16 +47,18 @@ WORKDIR /src RUN ln -s /usr/include /usr/local/include/netpbm -# Astrometry.net -RUN git clone http://github.com/dstndstn/astrometry.net.git astrometry \ - && cd astrometry \ - && make \ - && make py \ - && make extra \ - && make install INSTALL_DIR=/usr/local \ - && make clean && echo 2 +# Since this docker file is part of the repo, we directly map the repo folder to the container as the working directory. +# Note this is a one time mapping. Docker will pack the . folder (the repo folder) into a tar file and upload to the container. +# Due to some retrictions of Docker, we cannot directly use ../../, but have to specify the building context in the command line +# and refer it as . here. +# Note this doesn't include the index files, which are supposed to be downloaded locally and attach to the container through `docker run`. +ADD . /src/astrometry +RUN cd astrometry \ + && make -j \ + && make py -j \ + && make extra -j\ + && make install INSTALL_DIR=/usr/local # python = python3 RUN ln -s /usr/bin/python3 /usr/bin/python -ENV PYTHONPATH=/usr/local/lib/python - +ENV PYTHONPATH=/usr/local/lib/python \ No newline at end of file diff --git a/docker/webservice/Dockerfile b/docker/webservice/Dockerfile index 169eca520..48d345bc3 100644 --- a/docker/webservice/Dockerfile +++ b/docker/webservice/Dockerfile @@ -8,7 +8,16 @@ RUN apt -y update && \ apache2 \ libapache2-mod-wsgi-py3 \ less \ - emacs-nox + emacs-nox \ + ssh \ + openssh-server + +# Config the ssh server and client that are required for the web service +RUN ssh-keygen -q -t ed25519 -N '' -f ~/.ssh/id_ed25519 +RUN ssh-keygen -y -f ~/.ssh/id_ed25519 >> ~/.ssh/authorized_keys +RUN cp /src/astrometry/docker/webservice/config ~/.ssh/ +RUN service ssh start +RUN update-rc.d ssh defaults RUN pip3 install --no-cache-dir \ social-auth-core django-social-auth3 social-auth-app-django @@ -17,17 +26,10 @@ WORKDIR /src/astrometry/net RUN ln -s settings_test.py settings.py -# Yuck! The installed 'astrometry' package conflicts with '.', so paste it in... +# Yuck! The installed 'astrometry' package conflicts with '.', so paste it in... RUN rm -R /usr/local/lib/python/astrometry/net && \ ln -s /src/astrometry/net /usr/local/lib/python/astrometry/net -RUN mkdir appsecrets && \ - touch appsecrets/__init__.py && \ - touch appsecrets/auth.py -COPY django_db.py /src/astrometry/net/appsecrets/ - -RUN git stash && git pull - RUN mv migrations/* /tmp && \ python manage.py makemigrations && \ python manage.py migrate && \ @@ -36,6 +38,11 @@ RUN mv migrations/* /tmp && \ python manage.py loaddata fixtures/initial_data.json && \ python manage.py loaddata fixtures/flags.json -CMD python manage.py runserver +# Create the log dirs to avoid errors of not able to write to log files. +RUN mkdir /data +RUN mkdir /data/nova + +# Listen to all the bound addresses +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] EXPOSE 8000 diff --git a/docker/webservice/config b/docker/webservice/config new file mode 100755 index 000000000..d2818983b --- /dev/null +++ b/docker/webservice/config @@ -0,0 +1,8 @@ +Host an-test +Hostname 127.0.0.1 +User root +IdentityFile ~/.ssh/id_ed25519 +IdentitiesOnly yes +ForwardAgent no +ForwardX11 no +ForwardX11Trusted no \ No newline at end of file diff --git a/net/.gitignore b/net/.gitignore new file mode 100755 index 000000000..5cba93394 --- /dev/null +++ b/net/.gitignore @@ -0,0 +1 @@ +appsecrets \ No newline at end of file diff --git a/net/settings_test.py b/net/settings_test.py index 09a676750..e1a685583 100644 --- a/net/settings_test.py +++ b/net/settings_test.py @@ -4,6 +4,8 @@ ENABLE_SOCIAL = False +# Since this settings file is only for testing, disable the host restriction here to ensure a smooth deployment +ALLOWED_HOSTS = ['*'] DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3' DATABASES['default']['NAME'] = 'django.sqlite3' From 302721e0186e42c30694bf075a2dfafac45c116c Mon Sep 17 00:00:00 2001 From: Yan Wang Date: Sun, 1 Oct 2023 01:34:38 -0700 Subject: [PATCH 2/2] Revise the implementation according to discussions in https://github.com/dstndstn/astrometry.net/issues/283. --- docker/README.md | 12 +++++-- docker/solver/Dockerfile | 21 ++++++++---- docker/webservice/Dockerfile | 64 ++++++++++++++++++++++-------------- docker/webservice/config | 8 ----- net/launch.sh | 10 ++++++ net/nova-jobs.service | 2 +- 6 files changed, 74 insertions(+), 43 deletions(-) delete mode 100755 docker/webservice/config create mode 100755 net/launch.sh diff --git a/docker/README.md b/docker/README.md index e45bf1a47..077f134bc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -59,7 +59,15 @@ sudo docker build -t astrometrynet/webservice:latest -f docker/webservice/Docker For the container to function properly, we still need to map the indexes folder to it, with some port mapping: ``` -sudo docker run -p 8000:8000 -v ~/astrometry_indexes:/usr/local/data astrometrynet/webservice +sudo docker run -p 8000:8000 -v ~/astrometry_indexes:/data/INDEXES astrometrynet/webservice ``` -The the Astrometry.net website could be accessed on the host machine at http://localhost:8000. \ No newline at end of file +The the Astrometry.net website could be accessed on the host machine at http://localhost:8000. + +## Gap to Production + +Note the docker file still has quite some gap to production, especially in: + +1. All the data is stored in a SQLite "database," which is essentially a file and subject to loss after the container terminates. The solution is to create a "real" database somewhere, and let the django connect to it through the network. +2. Similarly, all the user uploaded data, results, and logs will be lost after the container terminates. The solution is to map a volume to `net/data`. +3. A good practice to handle many requests at the same time is to put the endpoint behind some reverse proxy with load balancing. Apache and Nginx are good candidates. \ No newline at end of file diff --git a/docker/solver/Dockerfile b/docker/solver/Dockerfile index 46431f174..9140b8491 100644 --- a/docker/solver/Dockerfile +++ b/docker/solver/Dockerfile @@ -34,30 +34,37 @@ RUN apt -y update && apt install -y apt-utils && \ python3-numpy \ python3-scipy \ python3-matplotlib \ + sudo \ && apt-get clean +# For both security reasons and compatibility with the service scripts (e.g. https://github.com/dstndstn/astrometry.net/blob/main/net/nova-jobs.service), we use a user nova here. +# Change the default password for better security here. +RUN useradd -m nova && echo "nova:changeme" | chpasswd && adduser nova sudo +RUN echo "nova ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN usermod -s /bin/bash nova + +RUN mkdir /home/nova/astrometry +WORKDIR /home/nova +RUN ln -s /usr/include /usr/local/include/netpbm + # Pip installs RUN for x in \ fitsio \ astropy \ ; do pip3 install --no-cache-dir $x; done -RUN mkdir /src -WORKDIR /src - -RUN ln -s /usr/include /usr/local/include/netpbm - # Since this docker file is part of the repo, we directly map the repo folder to the container as the working directory. # Note this is a one time mapping. Docker will pack the . folder (the repo folder) into a tar file and upload to the container. # Due to some retrictions of Docker, we cannot directly use ../../, but have to specify the building context in the command line # and refer it as . here. # Note this doesn't include the index files, which are supposed to be downloaded locally and attach to the container through `docker run`. -ADD . /src/astrometry +ADD . /home/nova/astrometry RUN cd astrometry \ && make -j \ && make py -j \ - && make extra -j\ + && make extra -j \ && make install INSTALL_DIR=/usr/local +RUN chown -R nova:nova /home/nova/astrometry # python = python3 RUN ln -s /usr/bin/python3 /usr/bin/python diff --git a/docker/webservice/Dockerfile b/docker/webservice/Dockerfile index 48d345bc3..108be65f3 100644 --- a/docker/webservice/Dockerfile +++ b/docker/webservice/Dockerfile @@ -1,7 +1,5 @@ FROM astrometrynet/solver:latest -RUN pip3 install --no-cache-dir Django - ENV DEBIAN_FRONTEND=noninteractive RUN apt -y update && \ apt install -y --no-install-recommends \ @@ -9,27 +7,37 @@ RUN apt -y update && \ libapache2-mod-wsgi-py3 \ less \ emacs-nox \ - ssh \ - openssh-server - -# Config the ssh server and client that are required for the web service -RUN ssh-keygen -q -t ed25519 -N '' -f ~/.ssh/id_ed25519 -RUN ssh-keygen -y -f ~/.ssh/id_ed25519 >> ~/.ssh/authorized_keys -RUN cp /src/astrometry/docker/webservice/config ~/.ssh/ -RUN service ssh start -RUN update-rc.d ssh defaults - -RUN pip3 install --no-cache-dir \ - social-auth-core django-social-auth3 social-auth-app-django - -WORKDIR /src/astrometry/net - -RUN ln -s settings_test.py settings.py + tmux \ + systemctl # Yuck! The installed 'astrometry' package conflicts with '.', so paste it in... RUN rm -R /usr/local/lib/python/astrometry/net && \ - ln -s /src/astrometry/net /usr/local/lib/python/astrometry/net + ln -s /home/nova/astrometry/net /usr/local/lib/python/astrometry/net +# Create the log dirs to avoid errors of not able to write to log files. +RUN mkdir /data && \ + mkdir /data/nova && \ + mkdir /data1 && \ + mkdir /data1/nova && \ + mkdir /data1/nova/tmp && \ + chown -R nova:nova /data && \ + chown -R nova:nova /data1 + +USER nova +RUN pip3 install --no-cache-dir \ + django \ + social-auth-core django-social-auth3 social-auth-app-django \ + astropy \ + fitsio \ + uwsgi + +WORKDIR /home/nova/astrometry/net +# For some reason need to recompile to resolve some weird GLIBC errors... +RUN cd .. && \ + make clean && \ + make -j && \ + make extra -j +RUN ln -s settings_test.py settings.py RUN mv migrations/* /tmp && \ python manage.py makemigrations && \ python manage.py migrate && \ @@ -38,11 +46,17 @@ RUN mv migrations/* /tmp && \ python manage.py loaddata fixtures/initial_data.json && \ python manage.py loaddata fixtures/flags.json -# Create the log dirs to avoid errors of not able to write to log files. -RUN mkdir /data -RUN mkdir /data/nova +USER root +RUN cd .. && make install INSTALL_DIR=/usr/local +RUN ln -s /home/nova/.local/bin/uwsgi /usr/local/bin/uwsgi +RUN cp nova-jobs.service /etc/systemd/system +RUN cp nova-uwsgi.service /etc/systemd/system +RUN systemctl enable nova-jobs +RUN systemctl enable nova-uwsgi +RUN systemctl start nova-jobs +RUN systemctl start nova-uwsgi -# Listen to all the bound addresses -CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] +USER nova +CMD ["/bin/bash", "./launch.sh"] -EXPOSE 8000 +EXPOSE 8000 \ No newline at end of file diff --git a/docker/webservice/config b/docker/webservice/config deleted file mode 100755 index d2818983b..000000000 --- a/docker/webservice/config +++ /dev/null @@ -1,8 +0,0 @@ -Host an-test -Hostname 127.0.0.1 -User root -IdentityFile ~/.ssh/id_ed25519 -IdentitiesOnly yes -ForwardAgent no -ForwardX11 no -ForwardX11Trusted no \ No newline at end of file diff --git a/net/launch.sh b/net/launch.sh new file mode 100755 index 000000000..595d96926 --- /dev/null +++ b/net/launch.sh @@ -0,0 +1,10 @@ +sudo systemctl start nova-jobs +sudo systemctl start nova-uwsgi + +# For the sake of a minimal deployment, we generate a very simple nova.cfg file on the fly. +# For more detailed usage of the cfg file, check /usr/local/astrometry.cfg or nova.cfg in net folder. +# Disable or modify this if more advanced features are demanded. +echo "add_path /data/INDEXES/" > nova.cfg +ls /data/INDEXES | awk '{printf("index %s\n", $1)}' >> nova.cfg + +python manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/net/nova-jobs.service b/net/nova-jobs.service index 8c8459960..92c44ef50 100644 --- a/net/nova-jobs.service +++ b/net/nova-jobs.service @@ -16,7 +16,7 @@ NotifyAccess=all WorkingDirectory=/home/nova/astrometry/net User=nova Environment="PYTHONPATH=/home/nova" "PATH=/usr/local/bin:/usr/bin:/bin:/home/nova/astrometry/util:/home/nova/astrometry/solver:/home/nova/astrometry/plot" -ExecStart=/usr/bin/python3 -u process_submissions.py -j 16 -s 4 --solve-locally=$(pwd)/solvescript.sh +ExecStart=/usr/bin/python3 -u process_submissions.py -j 16 -s 4 --solve-locally=/home/nova/astrometry/net/solvescript.sh #> /data/nova/dj.log 2>&1 [Install]