diff --git a/.d-compiler b/.d-compiler new file mode 100644 index 00000000..dac56262 --- /dev/null +++ b/.d-compiler @@ -0,0 +1 @@ +ldc-1.3.0 diff --git a/Procfile b/Procfile new file mode 100644 index 00000000..d96507ba --- /dev/null +++ b/Procfile @@ -0,0 +1,3 @@ +default_process_types: + web: ./dub-registry --port $PORT --mirror=https://code.dlang.org --hostname=dub-registry.heroku.com --separate-cron --vv --bind 0.0.0.0 + cron: ./dub-registry --mirror=https://code.dlang.org --run-cron --vv diff --git a/README.md b/README.md index 4f6aabe5..590e409f 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,5 @@ DUB registry ![vibe.d logo](public/images/logo-small.png) Online registry for [dub](https://github.com/dlang/dub/) packages, see . [![Build Status](https://travis-ci.org/dlang/dub-registry.svg)](https://travis-ci.org/dlang/dub-registry) + +[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/dlang/dub-registry) diff --git a/app.json b/app.json new file mode 100644 index 00000000..0e3f8d35 --- /dev/null +++ b/app.json @@ -0,0 +1,23 @@ +{ + "name": "DUB registry (mirror)", + "description": "A mirror of code.dlang.org", + "repository": "https://github.com/dub/dub-registry", + "website": "https://github.com/dub/dub-registry", + "logo": "https://dlang.org/images/dlogo_opengraph.png", + "keywords": ["d", "dlang", "dub", "mirror"], + "buildpacks": [ + { + "url": "http://github.com/MartinNowak/heroku-buildpack-d.git" + } + ], + "addons": [ + { + "plan": "mongolab:sandbox", + "as": "MONGO" + }, + { + "plan": "scheduler:standard", + "as": "SCHEDULER" + } + ] +} diff --git a/dub.sdl b/dub.sdl index c5e3e8a2..68f6d1ac 100644 --- a/dub.sdl +++ b/dub.sdl @@ -4,8 +4,8 @@ homepage "http://code.dlang.org/" authors "Sönke Ludwig" "Matthias Dondorff" license "GPL-3.0" -dependency "vibe-d" version="~>0.8.0-beta" -dependency "dub" version="~>1.3.0-beta" +dependency "vibe-d" version="~>0.8.0" +dependency "dub" version="~>1.3.0" dependency "userman" version="~>0.3.2" subConfiguration "dub" "library-nonet" @@ -14,7 +14,6 @@ versions "VibeJsonFieldNames" configuration "application" { targetType "executable" mainSourceFile "source/app.d" - versions "VibeDefaultMain" } configuration "library" { diff --git a/source/app.d b/source/app.d index cf16d841..0501dd98 100644 --- a/source/app.d +++ b/source/app.d @@ -26,14 +26,19 @@ DubRegistry s_registry; DubRegistryWebFrontend s_web; string s_mirror; +void checkForNewVersions() +{ + if (s_mirror.length) s_registry.mirrorRegistry(URL(s_mirror)); + else s_registry.checkForNewVersions(); +} + void startMonitoring() { void monitorNewVersions() { sleep(10.seconds()); // give the cache a chance to warm up first while(true){ - if (s_mirror.length) s_registry.mirrorRegistry(URL(s_mirror)); - else s_registry.checkForNewVersions(); + checkForNewVersions; sleep(30.minutes()); } } @@ -41,15 +46,25 @@ void startMonitoring() } version (linux) private immutable string certPath; - shared static this() +{ + enum debianCA = "/etc/ssl/certs/ca-certificates.crt"; + enum redhatCA = "/etc/pki/tls/certs/ca-bundle.crt"; + immutable certPath = redhatCA.exists ? redhatCA : debianCA; +} + +void main() { setLogFile("log.txt", LogLevel.diagnostic); string hostname = "code.dlang.org"; + bool separateCron; + bool runCron; readOption("mirror", &s_mirror, "URL of a package registry that this instance should mirror (WARNING: will overwrite local database!)"); readOption("hostname", &hostname, "Domain name of this instance (default: code.dlang.org)"); + readOption("separate-cron", &separateCron, "Use a separate cron job to query for packages."); + readOption("run-cron", &runCron, "Run cron operation."); // validate provided mirror URL if (s_mirror.length) @@ -57,10 +72,6 @@ shared static this() version (linux) { logInfo("Enforcing certificate trust."); - enum debianCA = "/etc/ssl/certs/ca-certificates.crt"; - enum redhatCA = "/etc/pki/tls/certs/ca-bundle.crt"; - certPath = redhatCA.exists ? redhatCA : debianCA; - HTTPClient.setTLSSetupCallback((ctx) { ctx.useTrustedCertificateFile(certPath); ctx.peerValidationMode = TLSPeerValidationMode.trustedCert; @@ -80,12 +91,20 @@ shared static this() auto router = new URLRouter; if (s_mirror.length) router.any("*", (req, res) { req.params["mirror"] = s_mirror; }); - router.get("*", (req, res) @trusted { if (!s_checkTask.running) startMonitoring(); }); + if (!separateCron) + router.get("*", (req, res) @trusted { if (!s_checkTask.running) startMonitoring(); }); // VPM registry + import dubregistry.mongodb : databaseName; auto regsettings = new DubRegistrySettings; + regsettings.databaseName = databaseName; s_registry = new DubRegistry(regsettings); + if (runCron) { + checkForNewVersions; + return; + } + UserManController userdb; if (!s_mirror.length) { @@ -115,5 +134,7 @@ shared static this() listenHTTP(settings, router); // poll github for new project versions - startMonitoring(); + if (!separateCron) + startMonitoring(); + runApplication(); } diff --git a/source/dubregistry/cache.d b/source/dubregistry/cache.d index 1970285a..177f0244 100644 --- a/source/dubregistry/cache.d +++ b/source/dubregistry/cache.d @@ -32,7 +32,8 @@ class URLCache { this() { - m_db = connectMongoDB("127.0.0.1"); + import dubregistry.mongodb : getMongoClient; + m_db = getMongoClient(); m_entries = m_db.getCollection("urlcache.entries"); m_entries.ensureIndex([tuple("url", 1)]); } @@ -70,7 +71,7 @@ class URLCache { // invalidate out of date cache entries if (be["_id"].get!BsonObjectID.timeStamp < now - m_maxCacheTime) m_entries.remove(["_id": be["_id"]]); - + deserializeBson(entry, be); if (mode == CacheMatchMode.always) { // directly return cache result for cache_priority == true diff --git a/source/dubregistry/dbcontroller.d b/source/dubregistry/dbcontroller.d index 6218a445..9d901617 100644 --- a/source/dubregistry/dbcontroller.d +++ b/source/dubregistry/dbcontroller.d @@ -23,7 +23,8 @@ class DbController { this(string dbname) { - auto db = connectMongoDB("127.0.0.1").getDatabase(dbname); + import dubregistry.mongodb : getMongoClient; + auto db = getMongoClient.getDatabase(dbname); m_packages = db["packages"]; m_downloads = db["downloads"]; diff --git a/source/dubregistry/mongodb.d b/source/dubregistry/mongodb.d new file mode 100644 index 00000000..74c6ab27 --- /dev/null +++ b/source/dubregistry/mongodb.d @@ -0,0 +1,24 @@ +module dubregistry.mongodb; + +import vibe.db.mongo.client : MongoClient; +import vibe.db.mongo.mongo : connectMongoDB; +import vibe.db.mongo.settings : MongoClientSettings, MongoAuthMechanism, parseMongoDBUrl; + +MongoClientSettings mongoSettings; +string databaseName = "vpmreg"; + +shared static this() +{ + import std.process : environment; + auto mongodbURI = environment.get("MONGODB_URI", "mongodb://127.0.0.1"); + parseMongoDBUrl(mongoSettings, mongodbURI); + mongoSettings.authMechanism = MongoAuthMechanism.scramSHA1; + if (mongoSettings.database.length != 0) + databaseName = mongoSettings.database; + mongoSettings.safe = true; +} + +MongoClient getMongoClient() +{ + return connectMongoDB(mongoSettings); +} diff --git a/source/dubregistry/notificationcenter.d b/source/dubregistry/notificationcenter.d index 411ca4f5..cde102ad 100644 --- a/source/dubregistry/notificationcenter.d +++ b/source/dubregistry/notificationcenter.d @@ -14,7 +14,8 @@ import userman.db.controller; import vibe.core.log; import vibe.mail.smtp; import vibe.stream.memory; -import vibe.templ.diet; +import vibe.stream.wrapper : StreamOutputRange; +import diet.html : compileHTMLDietFile; class NotificationCenter { @@ -52,7 +53,8 @@ class NotificationCenter { mail.headers["Subject"] = format("[%s] Errors in new version %s", package_name, branch_or_version); auto dst = createMemoryOutputStream(); - dst.compileDietFile!("dubregistry.mail.package-version-errors.dt", user, settings, package_name, branch_or_version, errors); + auto output = StreamOutputRange(dst); + output.compileHTMLDietFile!("dubregistry.mail.package-version-errors.dt", user, settings, package_name, branch_or_version, errors); mail.bodyText = cast(string)dst.data; sendMail(settings.mailSettings, mail); @@ -85,10 +87,11 @@ class NotificationCenter { mail.headers["Subject"] = format("Weekly deprecation warnings reminder"); auto dst = createMemoryOutputStream(); - dst.compileDietFile!("dubregistry.mail.package-deprecation-warnings.dt", user, settings, deprecations); + auto output = StreamOutputRange(dst); + output.compileHTMLDietFile!("dubregistry.mail.package-deprecation-warnings.dt", user, settings, deprecations); mail.bodyText = cast(string)dst.data; - sendMail(settings.mailSettings, mail); + //sendMail(settings.mailSettings, mail); } catch (Exception e) { logDiagnostic("Failed to send deprecation mail to %s <%s>: %s", user.fullName, user.email, e.msg); logDebug("Full error: %s", e.toString().sanitize); diff --git a/views/dubregistry.mail.package-version-errors.dt b/views/dubregistry.mail.package-version-errors.dt index ebd5ed25..95a14d3b 100644 --- a/views/dubregistry.mail.package-version-errors.dt +++ b/views/dubregistry.mail.package-version-errors.dt @@ -5,6 +5,7 @@ block title block body - import vibe.textfilter.urlencode; + - import std.algorithm.searching : startsWith; p The following errors have been detected for the recently added/updated version #{branch_or_version} of package #{package_name}: