exclude human photos from open data export #450

inaturalist · Jul 18, 2024 · 382d4a5 · 382d4a5
1 parent dc67bb0
commit 382d4a5
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 27 deletions.
diff --git a/lib/models/taxon.js b/lib/models/taxon.js
@@ -515,12 +515,14 @@ const Taxon = class Taxon extends Model {
   static async loadReferencedTaxa( ) {
     const query = squel.select( )
       .from( "taxa" )
-      .where( "is_active = ? AND name IN ?", true, ["Homo", "Life"] )
+      .where( "is_active = ? AND name IN ?", true, ["Homo", "Homo sapiens", "Life"] )
       .order( "id" );
     const { rows } = await pgClient.replica.query( query.toString( ) );
     _.each( rows, row => {
       if ( row.name === "Homo" && !Taxon.homo ) {
         Taxon.homo = row;
+      } else if ( row.name === "Homo sapiens" && !Taxon.homoSapiens ) {
+        Taxon.homoSapiens = row;
       } else if ( row.name === "Life" && !Taxon.life ) {
         Taxon.life = row;
       }

diff --git a/lib/open_data_archive.js b/lib/open_data_archive.js
@@ -10,6 +10,7 @@ const moment = require( "moment" );
 const { S3Client } = require( "@aws-sdk/client-s3" );
 const { Upload } = require( "@aws-sdk/lib-storage" );
 const Pool = require( "./pg_pool" );
+const Taxon = require( "./models/taxon" );
 const util = require( "./util" );
 const config = require( "../config" );
 
@@ -108,6 +109,7 @@ const OpenDataArchive = class OpenDataArchive {
   }
 
   async initialize( ) {
+    await Taxon.loadReferencedTaxa( );
     if ( _.isEmpty( PHOTO_DOMAIN ) ) {
       throw new Error( "config.aws.open_data_domain is undefined" );
     }
@@ -271,6 +273,8 @@ const OpenDataArchive = class OpenDataArchive {
       .field( "fe.extension" )
       .field( "p.user_id photo_user_id" )
       .field( "o.uuid obs_uuid" )
+      .field( "o.taxon_id obs_taxon_id" )
+      .field( "o.community_taxon_id obs_community_taxon_id" )
       .from( "observation_photos op" )
       .join( "photos p", null, "op.photo_id = p.id" )
       .join( "users u", null, "p.user_id = u.id" )
@@ -298,35 +302,43 @@ const OpenDataArchive = class OpenDataArchive {
     let lastObservationID = 0;
     let photoPosition = 0;
     _.each( rows, row => {
-      if ( row.prefix ) {
-        if ( row.observation_id === lastObservationID ) {
-          photoPosition += 1;
+      if ( !row.prefix ) {
+        return;
+      }
+      if ( row.obs_taxon_id === Taxon.homo.id
+        || row.obs_taxon_id === Taxon.homoSapiens.id
+        || row.obs_community_taxon_id === Taxon.homo.id
+        || row.obs_community_taxon_id === Taxon.homoSapiens.id
+      ) {
+        return;
+      }
+      if ( row.observation_id === lastObservationID ) {
+        photoPosition += 1;
+      } else {
+        photoPosition = 0;
+      }
+      if ( row.prefix.match( photoRegex ) ) {
+        if ( VALID_EXTENSIONS[_.toLower( row.extension )] ) {
+          const photoFileFields = [
+            row.photo_uuid,
+            row.photo_id,
+            row.obs_uuid,
+            row.photo_user_id,
+            row.extension,
+            license( row.license ),
+            row.width,
+            row.height,
+            photoPosition
+          ];
+          this.photoFileStream.write( `${photoFileFields.join( "\t" )}\n` );
+          userIDs[row.photo_user_id] = true;
+          observationIDs[row.observation_id] = true;
         } else {
-          photoPosition = 0;
-        }
-        if ( row.prefix.match( photoRegex ) ) {
-          if ( VALID_EXTENSIONS[_.toLower( row.extension )] ) {
-            const photoFileFields = [
-              row.photo_uuid,
-              row.photo_id,
-              row.obs_uuid,
-              row.photo_user_id,
-              row.extension,
-              license( row.license ),
-              row.width,
-              row.height,
-              photoPosition
-            ];
-            this.photoFileStream.write( `${photoFileFields.join( "\t" )}\n` );
-            userIDs[row.photo_user_id] = true;
-            observationIDs[row.observation_id] = true;
-          } else {
-            // invalid extension
-            // console.log( extension );
-          }
+          // invalid extension
+          // console.log( extension );
         }
-        lastObservationID = row.observation_id;
       }
+      lastObservationID = row.observation_id;
     } );
 
     await this.fetchObservations(

diff --git a/lib/tasks/open_data_export.js b/lib/tasks/open_data_export.js
@@ -17,5 +17,7 @@ setTimeout( async ( ) => {
       console.log( e );
     } ).finally( ( ) => {
       console.log( "" );
+      console.log( "we're done" );
+      process.exit( );
     } );
 }, 2000 );