Fixing the top level zcl file registration when the gsdk is updated f…

…rom one branch to another - There is a check for whether a zcl.json or any of its child xml packages being updated. If updated, then the top level package is added again along with its child packages with the is_in_sync column as true for the top level package. However the old top level package's is_in_sync is turned to false to point out an outdated package. - All the query-pacakge queries have been updated to use the is_in_sync = 1 package such that the old one is not picked up by mistake. - Updated the zap schema, added is_in_sync to package table. - Adding unit tests for checking gsdk upgrades through xml file updates. Making sure that all zcl packages are reloaded when there is a change to any. - JRIA: ZAPP-1079
project-chip · Jun 26, 2024 · 26feb6e · 26feb6e
1 parent 078ee25
commit 26feb6e
Show file tree

Hide file tree

Showing 9 changed files with 4,558 additions and 4,171 deletions.
diff --git a/docs/zap-schema.svg b/docs/zap-schema.svg
diff --git a/src-electron/db/query-package.js b/src-electron/db/query-package.js
@@ -100,10 +100,11 @@ async function getZclPropertiesPackage(db, packages) {
  */
 async function getPackageByPathAndType(db, path, type) {
   return dbApi
-    .dbGet(db, `${querySelectFromPackage} WHERE PATH = ? AND TYPE = ?`, [
-      path,
-      type,
-    ])
+    .dbGet(
+      db,
+      `${querySelectFromPackage} WHERE PATH = ? AND TYPE = ? AND IS_IN_SYNC = 1`,
+      [path, type]
+    )
     .then(dbMapping.map.package)
 }
 
@@ -119,8 +120,18 @@ async function getPackageByPathAndType(db, path, type) {
 async function getPackageIdByPathAndTypeAndVersion(db, path, type, version) {
   // Version can be null for custom xml
   let packageQuery =
-    `SELECT PACKAGE_ID FROM PACKAGE WHERE PATH = '${path}' AND TYPE = '${type}' AND ` +
-    (version ? `VERSION = '${version}'` : `VERSION IS NULL`)
+    `
+    SELECT
+      PACKAGE_ID
+    FROM
+      PACKAGE
+    WHERE
+      IS_IN_SYNC = 1
+    AND
+      PATH = '${path}'
+    AND
+      TYPE = '${type}'
+    AND ` + (version ? `VERSION = '${version}'` : `VERSION IS NULL`)
   return dbApi.dbGet(db, packageQuery).then((row) => {
     if (row == null) return null
     else return row.PACKAGE_ID
@@ -136,7 +147,9 @@ async function getPackageIdByPathAndTypeAndVersion(db, path, type, version) {
  */
 async function getPackagesByType(db, type) {
   return dbApi
-    .dbAll(db, `${querySelectFromPackage} WHERE TYPE = ?`, [type])
+    .dbAll(db, `${querySelectFromPackage} WHERE TYPE = ? AND IS_IN_SYNC = 1`, [
+      type,
+    ])
     .then((rows) => rows.map(dbMapping.map.package))
 }
 /**
@@ -150,7 +163,7 @@ async function getPackagesByCategoryAndType(db, type, category = '') {
   return dbApi
     .dbAll(
       db,
-      `${querySelectFromPackage} WHERE TYPE = ? AND (CATEGORY IN (${category}) OR CATEGORY IS NULL)`,
+      `${querySelectFromPackage} WHERE IS_IN_SYNC = 1 AND TYPE = ? AND (CATEGORY IN (${category}) OR CATEGORY IS NULL)`,
       [type]
     )
     .then((rows) => rows.map(dbMapping.map.package))
@@ -231,16 +244,20 @@ async function getPackageRefByAttributeId(db, attributeId) {
  * @returns Promise resolving with a CRC or null.
  */
 async function getPathCrc(db, path) {
-  return dbApi.dbGet(db, 'SELECT CRC FROM PACKAGE WHERE PATH = ?', [path]).then(
-    (row) =>
-      new Promise((resolve, reject) => {
-        if (row == null) {
-          resolve(null)
-        } else {
-          resolve(row.CRC)
-        }
-      })
-  )
+  return dbApi
+    .dbGet(db, 'SELECT CRC FROM PACKAGE WHERE PATH = ? AND IS_IN_SYNC = 1', [
+      path,
+    ])
+    .then(
+      (row) =>
+        new Promise((resolve, reject) => {
+          if (row == null) {
+            resolve(null)
+          } else {
+            resolve(row.CRC)
+          }
+        })
+    )
 }
 
 /**
@@ -300,10 +317,11 @@ async function registerTopLevelPackage(
   type,
   version = null,
   category = null,
-  description = null
+  description = null,
+  isTopLevelPackageInSync = true
 ) {
   let row = await getPackageByPathAndType(db, path, type)
-  if (row == null) {
+  if (row == null || !isTopLevelPackageInSync) {
     // Doesn't exist. We have to add it.
     let id = await dbApi.dbInsert(
       db,
@@ -339,7 +357,22 @@ async function updatePathCrc(db, path, crc, parentId) {
   return dbApi.dbUpdate(
     db,
     'UPDATE PACKAGE SET CRC = ? WHERE PATH = ? AND PARENT_PACKAGE_REF = ?',
-    [path, crc, parentId]
+    [crc, path, parentId]
+  )
+}
+
+/**
+ * Updates a is in sync in the package table.
+ * @param {*} db
+ * @param {*} packageRef
+ * @param {*} isInSync
+ * @returns Promise of an update.
+ */
+async function updatePackageIsInSync(db, packageRef, isInSync) {
+  return dbApi.dbUpdate(
+    db,
+    'UPDATE PACKAGE SET IS_IN_SYNC = ? WHERE PACKAGE_ID = ?',
+    [dbApi.toDbBool(isInSync), packageRef]
   )
 }
 
@@ -650,8 +683,10 @@ async function getAllPackages(db) {
         CATEGORY,
         DESCRIPTION,
         PARENT_PACKAGE_REF
-       FROM
-        PACKAGE`
+       FROM 
+        PACKAGE
+       WHERE
+        IS_IN_SYNC = 1`
     )
     .then((rows) => rows.map(dbMapping.map.package))
 }
@@ -1117,3 +1152,4 @@ exports.insertSessionKeyValuesFromPackageDefaults =
 exports.getPackagesByCategoryAndType = getPackagesByCategoryAndType
 exports.getPackagesByPackageIds = getPackagesByPackageIds
 exports.getPackageByPathAndType = getPackageByPathAndType
+exports.updatePackageIsInSync = updatePackageIsInSync
diff --git a/src-electron/db/zap-schema.sql b/src-electron/db/zap-schema.sql
@@ -36,6 +36,7 @@ CREATE TABLE "PACKAGE" (
   "VERSION" integer,
   "CATEGORY" text,
   "DESCRIPTION" text,
+  "IS_IN_SYNC" boolean default 1,
   foreign key (PARENT_PACKAGE_REF) references PACKAGE(PACKAGE_ID) ON DELETE CASCADE ON UPDATE CASCADE
 );
 /*

diff --git a/src-electron/generator/generation-engine.js b/src-electron/generator/generation-engine.js
@@ -124,7 +124,8 @@ async function recordTemplatesPackage(context) {
     dbEnum.packageType.genTemplatesJson,
     context.templateData.version,
     context.templateData.category,
-    context.templateData.description
+    context.templateData.description,
+    true
   )
   context.packageId = topLevel.id
   if (topLevel.existedPreviously) return context

diff --git a/src-electron/util/env.js b/src-electron/util/env.js
@@ -35,6 +35,14 @@ export function builtinSilabsZclMetafile() {
   return locateProjectResource('./zcl-builtin/silabs/zcl.json')
 }
 
+/**
+ *
+ * @returns path to general.xml file
+ */
+export function builtinSilabsZclGeneralXmlFile() {
+  return locateProjectResource('./zcl-builtin/silabs/general.xml')
+}
+
 export function builtinMatterZclMetafile() {
   return locateProjectResource('./zcl-builtin/matter/zcl.json')
 }

diff --git a/src-electron/zcl/zcl-loader-silabs.js b/src-electron/zcl/zcl-loader-silabs.js
@@ -1983,6 +1983,55 @@ async function parseSingleZclFile(db, packageId, file, context) {
   }
 }
 
+/**
+ * Checks if there is a crc mismatch on any xml file. This can be used to
+ * decide if there is a need to reload all the xml files. Also check if the
+ * package is not loaded before.
+ * @param {*} db
+ * @param {*} packageId
+ * @param {*} files
+ * @returns the status of crc mismatch and whether a package is present in an
+ * object
+ */
+async function isCrcMismatchOrPackageDoesNotExist(db, packageId, files) {
+  let packagesNotFound = 0
+  let packagesFound = 0
+  let result = { isCrcMismatch: false, areSomePackagesNotLoaded: false }
+  for (let file of files) {
+    let fileContent = await fsp.readFile(file)
+    let filePath = file
+    let actualCrc = util.checksum(fileContent)
+
+    let pkg = await queryPackage.getPackageByPathAndParent(
+      db,
+      filePath,
+      packageId,
+      false
+    )
+
+    if (pkg != null && pkg.crc != actualCrc) {
+      env.logDebug(
+        `CRC missmatch for file ${pkg.path}, (${pkg.crc} vs ${actualCrc}) package id ${pkg.id}, parsing.
+        Mismatch with package id: ${packageId}`
+      )
+      result.isCrcMismatch = true
+      return result
+    } else if (pkg == null) {
+      // This is executed if there is no CRC in the database.
+      packagesNotFound++
+      env.logDebug(
+        `No CRC in the database for file ${filePath}. Package needs to be loaded`
+      )
+    } else if (pkg != null && pkg.crc == actualCrc) {
+      packagesFound++
+    }
+  }
+  result.areSomePackagesNotLoaded = !(
+    packagesNotFound == files.length || packagesFound == files.length
+  )
+  return result
+}
+
 /**
  *
  * Promises to iterate over all the XML files and returns an aggregate promise
@@ -2473,18 +2522,34 @@ async function loadZclJsonOrProperties(db, metafile, isJson = false) {
 
   try {
     Object.assign(ctx, await util.readFileContentAndCrc(ctx.metadataFile))
-    ctx.packageId = await zclLoader.recordToplevelPackage(
-      db,
-      ctx.metadataFile,
-      ctx.crc
-    )
     let ret
     if (isJson) {
       ret = await collectDataFromJsonFile(ctx.metadataFile, ctx.data)
     } else {
       ret = await collectDataFromPropertiesFile(ctx.metadataFile, ctx.data)
     }
     Object.assign(ctx, ret)
+    ctx.packageId = await zclLoader.recordToplevelPackage(
+      db,
+      ctx.metadataFile,
+      ctx.crc,
+      true
+    )
+    let packageStatus = await isCrcMismatchOrPackageDoesNotExist(
+      db,
+      ctx.packageId,
+      ctx.zclFiles
+    )
+    if (packageStatus.isCrcMismatch || packageStatus.areSomePackagesNotLoaded) {
+      await queryPackage.updatePackageIsInSync(db, ctx.packageId, 0)
+      ctx.packageId = await zclLoader.recordToplevelPackage(
+        db,
+        ctx.metadataFile,
+        ctx.crc,
+        false
+      )
+    }
+
     if (
       ctx.version != null ||
       ctx.category != null ||

diff --git a/src-electron/zcl/zcl-loader.js b/src-electron/zcl/zcl-loader.js
@@ -36,14 +36,24 @@ const queryNotification = require('../db/query-session-notification')
  * @param {*} db
  * @param {*} metadataFile
  * @param {*} crc
+ * @param {*} isTopLevelPackageInSync
  * @returns packageId
  */
-async function recordToplevelPackage(db, metadataFile, crc) {
+async function recordToplevelPackage(
+  db,
+  metadataFile,
+  crc,
+  isTopLevelPackageInSync
+) {
   let topLevel = await queryPackage.registerTopLevelPackage(
     db,
     metadataFile,
     crc,
-    dbEnum.packageType.zclProperties
+    dbEnum.packageType.zclProperties,
+    null,
+    null,
+    null,
+    isTopLevelPackageInSync
   )
   return topLevel.id
 }
@@ -269,6 +279,7 @@ async function loadIndividualFile(db, filePath, sessionId) {
  * @param {*} db
  * @param {*} info
  * @param {*} parentPackageId
+ * @param {*} isCustom
  * @returns Promise that resolves int he object of data.
  */
 async function qualifyZclFile(

diff --git a/test/zcl-loader.test.js b/test/zcl-loader.test.js
@@ -29,6 +29,7 @@ const env = require('../src-electron/util/env')
 const types = require('../src-electron/util/types')
 const testUtil = require('./test-util')
 const testQuery = require('./test-query')
+const fs = require('fs')
 
 beforeAll(async () => {
   env.setDevelopmentEnv()
@@ -188,6 +189,50 @@ test(
           `Found Non Unique Enum in Silabs XML: ${c.NAME} ${c.TYPE} ${c.PACKAGE_REF}`
         )
       })
+
+      // Test Reloading of the zcl packages when an xml file changes
+      // (Simulating a gsdk upgrade from one branch to another which can change xml files)
+      // Step 1: Modify one of the xml files
+      let xmlFilePath = env.builtinSilabsZclGeneralXmlFile()
+      let originalString =
+        '<attribute side="server" code="0x0001" define="APPLICATION_VERSION" type="INT8U" min="0x00" max="0xFF" writable="false" default="0x00" optional="true">application version</attribute>'
+      let editString =
+        '<attribute side="server" code="0x0001" define="APPLICATION_VERSION" type="INT8U" min="0x00" max="0xFF" writable="false" default="0x01" optional="true">application version</attribute>'
+      let generalXmlFileOriginalContent = fs.readFileSync(xmlFilePath, 'utf8')
+      let generalXmlFileUpdatedContent = generalXmlFileOriginalContent.replace(
+        originalString,
+        editString
+      )
+      fs.writeFileSync(xmlFilePath, generalXmlFileUpdatedContent, 'utf8')
+
+      // Step 2: This will cause the all zcl packages from top level to all its
+      // children to be reloaded. Check for 2 top level packages with same path now.
+      // Count packages belonging to each top level package to make sure all packages
+      // are reloaded
+      ctx = await zclLoader.loadZcl(db, env.builtinSilabsZclMetafile())
+      let newPackageId = ctx.packageId
+      // Making sure the top level packages do not have the same packageId
+      expect(packageId).not.toEqual(newPackageId)
+      let oldPackages = await dbApi.dbAll(
+        db,
+        `SELECT * FROM PACKAGE WHERE PARENT_PACKAGE_REF = ${packageId}`
+      )
+      let newPackages = await dbApi.dbAll(
+        db,
+        `SELECT * FROM PACKAGE WHERE PARENT_PACKAGE_REF = ${newPackageId}`
+      )
+      // Making sure all packages are loaded again
+      expect(oldPackages.length).toEqual(newPackages.length)
+
+      let topLevelZclPackages = await dbApi.dbAll(
+        db,
+        `SELECT * FROM PACKAGE WHERE TYPE = '${dbEnum.packageType.zclProperties}' ORDER BY PACKAGE_ID`
+      )
+      expect(topLevelZclPackages[0].IS_IN_SYNC).toEqual(0)
+      expect(topLevelZclPackages[1].IS_IN_SYNC).toEqual(1)
+
+      // Step 3: Revert the xml file change in step 1.
+      fs.writeFileSync(xmlFilePath, generalXmlFileOriginalContent, 'utf8')
     } finally {
       await dbApi.closeDatabase(db)
     }