Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DPO3DPKRT-763/generate download scenes workaround #552

Merged
merged 7 commits into from
Oct 10, 2023
Merged

DPO3DPKRT-763/generate download scenes workaround #552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0dd3bcc
Temporarily bypass Generate Downloads while development wraps up
EMaslowskiQ Oct 6, 2023
262971c
Merge commit '785f292487817b6bcb166205b522629d5ce060a1'
EMaslowskiQ Oct 6, 2023
11d8878
updated EDAN default derver for development.
EMaslowskiQ Oct 6, 2023
14ef258
remove additional reference to generating downloads
EMaslowskiQ Oct 9, 2023
1e81faf
cleanup of unused variables due to temporary fix
EMaslowskiQ Oct 9, 2023
0aa1030
fixed conditional logic when ingesting and added log output
EMaslowskiQ Oct 9, 2023
5a967b2
Merge branch 'develop' into DPO3DPKRT-763/hotfix-generate-download-sc…
EMaslowskiQ Oct 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions server/collections/impl/PublishScene.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ export class PublishScene {
return DownloadMSXMap;
}

static async handleSceneUpdates(idScene: number, idSystemObject: number, idUser: number | undefined,
static async handleSceneUpdates(idScene: number, idSystemObject: number, _idUser: number | undefined,
oldPosedAndQCd: boolean, newPosedAndQCd: boolean,
LicenseOld: DBAPI.License | undefined, LicenseNew: DBAPI.License | undefined): Promise<SceneUpdateResult> {
// if we've changed Posed and QC'd, and/or we've updated our license, create or remove downloads
Expand All @@ -303,12 +303,14 @@ export class PublishScene {
return PublishScene.sendResult(true);

if (newDownloadState) {
LOG.info(`PublishScene.handleSceneUpdates generating downloads for scene ${idScene}`, LOG.LS.eGQL);
LOG.info(`PublishScene.handleSceneUpdates generating downloads for scene ${idScene} (skipping)`, LOG.LS.eGQL);
// Generate downloads
const workflowEngine: WF.IWorkflowEngine | null = await WF.WorkflowFactory.getInstance();
if (!workflowEngine)
return PublishScene.sendResult(false, `Unable to fetch workflow engine for download generation for scene ${idScene}`);
workflowEngine.generateSceneDownloads(idScene, { idUserInitiator: idUser }); // don't await

// HACK: temporarily skip generate downloads while development on that wraps up
// workflowEngine.generateSceneDownloads(idScene, { idUserInitiator: idUser }); // don't await
return { success: true, downloadsGenerated: true, downloadsRemoved: false };
} else { // Remove downloads
LOG.info(`PublishScene.handleSceneUpdates removing downloads for scene ${idScene}`, LOG.LS.eGQL);
Expand Down
2 changes: 1 addition & 1 deletion server/config/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ export const Config: ConfigType = {
type: COLLECTION_TYPE.EDAN,
edan: {
// defaulting to development environment for EDAN if environment variables not set for safety.
server: process.env.PACKRAT_EDAN_SERVER ? process.env.PACKRAT_EDAN_SERVER : /* istanbul ignore next */ 'https://edandev.si.edu/',
server: process.env.PACKRAT_EDAN_SERVER ? process.env.PACKRAT_EDAN_SERVER : /* istanbul ignore next */ 'https://edan.si.edu/',
api3d: process.env.PACKRAT_EDAN_3D_API ? process.env.PACKRAT_EDAN_3D_API : /* istanbul ignore next */ 'https://console.si.edu/apis/3d-api-dev/',
appId: process.env.PACKRAT_EDAN_APPID ? process.env.PACKRAT_EDAN_APPID : /* istanbul ignore next */ 'OCIO3D',
authKey: process.env.PACKRAT_EDAN_AUTH_KEY ? process.env.PACKRAT_EDAN_AUTH_KEY : /* istanbul ignore next */ '',
Expand Down
145 changes: 78 additions & 67 deletions server/workflow/impl/Packrat/WorkflowEngine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ type ComputeSceneInfoResult = {
export class WorkflowEngine implements WF.IWorkflowEngine {
private workflowMap: Map<number, WF.IWorkflow> = new Map<number, WF.IWorkflow>();

// HACK: temporarily bypass download generation while development on it wraps up.
private tmpSkipGenDownloads: boolean = true;

async create(workflowParams: WF.WorkflowParameters): Promise<WF.IWorkflow | null> {
if (!workflowParams.eWorkflowType) {
LOG.error(`WorkflowEngine.create called without workflow type ${JSON.stringify(workflowParams)}`, LOG.LS.eWF);
Expand Down Expand Up @@ -264,7 +267,6 @@ export class WorkflowEngine implements WF.IWorkflowEngine {
return null;
}


const SOGeometry: DBAPI.SystemObject| null = await CMIR.assetVersionGeometry.fetchSystemObject();
if (!SOGeometry) {
LOG.error(`WorkflowEngine.eventIngestionIngestObjectModel unable to compute geometry file systemobject from ${JSON.stringify(CMIR.assetVersionGeometry, H.Helpers.saferStringify)}`, LOG.LS.eWF);
Expand Down Expand Up @@ -313,58 +315,62 @@ export class WorkflowEngine implements WF.IWorkflowEngine {
} else
LOG.info(`WorkflowEngine.eventIngestionIngestObjectModel skipping si-voyager-scene for master model with unsupported units ${JSON.stringify(CMIR, H.Helpers.saferStringify)}`, LOG.LS.eWF);

// does this ingested model have a scene child? If so, initiate WorkflowJob for cook si-generate-downloads
const SODerived: DBAPI.SystemObject[] | null = CMIR.idSystemObjectModel ? await DBAPI.SystemObject.fetchDerivedFromXref(CMIR.idSystemObjectModel) : null;
if (!SODerived)
return workflows.length > 0 ? workflows : null;

for (const SO of SODerived) {
if (SO.idScene) {
// locate the scene file asset attached to this system object
const sceneAssetVersions: DBAPI.AssetVersion[] | null = await DBAPI.AssetVersion.fetchFromSystemObject(SO.idSystemObject);
if (!sceneAssetVersions) {
LOG.error(`WorkflowEngine.eventIngestionIngestObjectModel unable to fetch scene's assets from ${JSON.stringify(SO, H.Helpers.saferStringify)}`, LOG.LS.eWF);
continue;
}
// HACK: skip generate downloads while other issues are resolved with it
if(this.tmpSkipGenDownloads===false) {
// does this ingested model have a scene child? If so, initiate WorkflowJob for cook si-generate-downloads
const SODerived: DBAPI.SystemObject[] | null = CMIR.idSystemObjectModel ? await DBAPI.SystemObject.fetchDerivedFromXref(CMIR.idSystemObjectModel) : null;
if (!SODerived)
return workflows.length > 0 ? workflows : null;

for (const SO of SODerived) {
if (SO.idScene) {
// locate the scene file asset attached to this system object
const sceneAssetVersions: DBAPI.AssetVersion[] | null = await DBAPI.AssetVersion.fetchFromSystemObject(SO.idSystemObject);
if (!sceneAssetVersions) {
LOG.error(`WorkflowEngine.eventIngestionIngestObjectModel unable to fetch scene's assets from ${JSON.stringify(SO, H.Helpers.saferStringify)}`, LOG.LS.eWF);
continue;
}

let sceneAssetVersion: DBAPI.AssetVersion | null = null;
for (const AV of sceneAssetVersions) {
if (AV.FileName.toLowerCase().endsWith('.svx.json')) {
sceneAssetVersion = AV;
break;
let sceneAssetVersion: DBAPI.AssetVersion | null = null;
for (const AV of sceneAssetVersions) {
if (AV.FileName.toLowerCase().endsWith('.svx.json')) {
sceneAssetVersion = AV;
break;
}
}
}

if (!sceneAssetVersion) {
LOG.error(`WorkflowEngine.eventIngestionIngestObjectModel unable to find scene .svx.json's asset version from ${JSON.stringify(SO, H.Helpers.saferStringify)}`, LOG.LS.eWF);
continue;
}
if (!sceneAssetVersion) {
LOG.error(`WorkflowEngine.eventIngestionIngestObjectModel unable to find scene .svx.json's asset version from ${JSON.stringify(SO, H.Helpers.saferStringify)}`, LOG.LS.eWF);
continue;
}

const idSystemObjectClone: number[] = L.clone(idSystemObject);
const SOSceneAsset: DBAPI.SystemObject | null = sceneAssetVersion ? await sceneAssetVersion.fetchSystemObject() : null;
if (SOSceneAsset)
idSystemObjectClone.push(SOSceneAsset.idSystemObject);

const idSystemObjectClone: number[] = L.clone(idSystemObject);
const SOSceneAsset: DBAPI.SystemObject | null = sceneAssetVersion ? await sceneAssetVersion.fetchSystemObject() : null;
if (SOSceneAsset)
idSystemObjectClone.push(SOSceneAsset.idSystemObject);

const jobParamSIGenerateDownloads: WFP.WorkflowJobParameters =
new WFP.WorkflowJobParameters(COMMON.eVocabularyID.eJobJobTypeCookSIGenerateDownloads,
new COOK.JobCookSIGenerateDownloadsParameters(SO.idScene, CMIR.idModel, CMIR.assetVersionGeometry.FileName,
sceneAssetVersion.FileName, CMIR.assetVersionDiffuse?.FileName, CMIR.assetVersionMTL?.FileName, modelBaseName));

const wfParamSIGenerateDownloads: WF.WorkflowParameters = {
eWorkflowType: COMMON.eVocabularyID.eWorkflowTypeCookJob,
idSystemObject: idSystemObjectClone,
idProject: workflowParams.idProject,
idUserInitiator: workflowParams.idUserInitiator,
parameters: jobParamSIGenerateDownloads,
};

const wfSIGenerateDownloads: WF.IWorkflow | null = await this.create(wfParamSIGenerateDownloads);
if (wfSIGenerateDownloads)
workflows.push(wfSIGenerateDownloads);
else
LOG.error(`WorkflowEngine.eventIngestionUploadAssetVersion unable to create Cook si-voyager-scene workflow: ${JSON.stringify(wfParamSIGenerateDownloads)}`, LOG.LS.eWF);
const jobParamSIGenerateDownloads: WFP.WorkflowJobParameters =
new WFP.WorkflowJobParameters(COMMON.eVocabularyID.eJobJobTypeCookSIGenerateDownloads,
new COOK.JobCookSIGenerateDownloadsParameters(SO.idScene, CMIR.idModel, CMIR.assetVersionGeometry.FileName,
sceneAssetVersion.FileName, CMIR.assetVersionDiffuse?.FileName, CMIR.assetVersionMTL?.FileName, modelBaseName));

const wfParamSIGenerateDownloads: WF.WorkflowParameters = {
eWorkflowType: COMMON.eVocabularyID.eWorkflowTypeCookJob,
idSystemObject: idSystemObjectClone,
idProject: workflowParams.idProject,
idUserInitiator: workflowParams.idUserInitiator,
parameters: jobParamSIGenerateDownloads,
};

const wfSIGenerateDownloads: WF.IWorkflow | null = await this.create(wfParamSIGenerateDownloads);
if (wfSIGenerateDownloads)
workflows.push(wfSIGenerateDownloads);
else
LOG.error(`WorkflowEngine.eventIngestionUploadAssetVersion unable to create Cook si-voyager-scene workflow: ${JSON.stringify(wfParamSIGenerateDownloads)}`, LOG.LS.eWF);
}
}
}
} else
LOG.info('WorkflowEngine.eventIngestionUploadAssetVersion skipping generating downloads',LOG.LS.eWF);

return workflows.length > 0 ? workflows : null;
}
Expand Down Expand Up @@ -402,25 +408,30 @@ export class WorkflowEngine implements WF.IWorkflowEngine {
if (SOMTL)
idSystemObject.push(SOMTL.idSystemObject);

// initiate WorkflowJob for cook si-generate-download
const { modelBaseName } = await WorkflowEngine.computeSceneAndModelBaseNames(CSIR.idModel, CSIR.assetVersionGeometry.FileName);
const jobParamSIGenerateDownloads: WFP.WorkflowJobParameters =
new WFP.WorkflowJobParameters(COMMON.eVocabularyID.eJobJobTypeCookSIGenerateDownloads,
new COOK.JobCookSIGenerateDownloadsParameters(CSIR.idScene, CSIR.idModel, CSIR.assetVersionGeometry.FileName,
CSIR.assetSVX.FileName, CSIR.assetVersionDiffuse?.FileName, CSIR.assetVersionMTL?.FileName, modelBaseName));

const wfParamSIGenerateDownloads: WF.WorkflowParameters = {
eWorkflowType: COMMON.eVocabularyID.eWorkflowTypeCookJob,
idSystemObject,
idProject: workflowParams.idProject,
idUserInitiator: workflowParams.idUserInitiator,
parameters: jobParamSIGenerateDownloads,
};

const workflow: WF.IWorkflow | null = await this.create(wfParamSIGenerateDownloads);
if (workflow)
return [workflow];
LOG.error(`WorkflowEngine.eventIngestionIngestObjectScene unable to create Cook si-generate-downloads workflow: ${JSON.stringify(wfParamSIGenerateDownloads)}`, LOG.LS.eWF);
// HACK: skip generate downloads while other issues are resolved with it
if(this.tmpSkipGenDownloads===false) {
// initiate WorkflowJob for cook si-generate-download
const { modelBaseName } = await WorkflowEngine.computeSceneAndModelBaseNames(CSIR.idModel, CSIR.assetVersionGeometry.FileName);
const jobParamSIGenerateDownloads: WFP.WorkflowJobParameters =
new WFP.WorkflowJobParameters(COMMON.eVocabularyID.eJobJobTypeCookSIGenerateDownloads,
new COOK.JobCookSIGenerateDownloadsParameters(CSIR.idScene, CSIR.idModel, CSIR.assetVersionGeometry.FileName,
CSIR.assetSVX.FileName, CSIR.assetVersionDiffuse?.FileName, CSIR.assetVersionMTL?.FileName, modelBaseName));

const wfParamSIGenerateDownloads: WF.WorkflowParameters = {
eWorkflowType: COMMON.eVocabularyID.eWorkflowTypeCookJob,
idSystemObject,
idProject: workflowParams.idProject,
idUserInitiator: workflowParams.idUserInitiator,
parameters: jobParamSIGenerateDownloads,
};

const workflow: WF.IWorkflow | null = await this.create(wfParamSIGenerateDownloads);
if (workflow)
return [workflow];
LOG.error(`WorkflowEngine.eventIngestionIngestObjectScene unable to create Cook si-generate-downloads workflow: ${JSON.stringify(wfParamSIGenerateDownloads)}`, LOG.LS.eWF);
} else
LOG.info('WorkflowEngine.eventIngestionUploadAssetVersion skipping generating downloads',LOG.LS.eWF);

return null;
}

Expand Down
Loading