Merge pull request #4 from Azure/master

merge 4

README.md

@@ -8,7 +8,7 @@ There are two processes for updating schemas:
 2. **Manually**: Schemas are manually authored and committed via PR.
 
 ### Has my team been onboarded for daily autogeneration?
-Please see [generator/whitelist.ts](/generator/whitelist.ts) for the list of teams which have been onboarded. `basePath` refers to the path in the [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs) repo, and `namespace` is the Resource Provider namespace.
+Please see [generator/autogenlist.ts](/generator/autogenlist.ts) for the list of teams which have been onboarded. `basePath` refers to the path in the [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs) repo, and `namespace` is the Resource Provider namespace.
 
 **If your team has been onboarded, we do not require any manual contributions to this repo and your schemas will automatically be kept up to date by the pipeline.**
 

generator/README.md

@@ -9,7 +9,7 @@
 * `npm run clean` - Resets the local schemas directory to a clean state.
 * `npm run list-basepaths` - Lists all the valid base paths from the specs repo for generation.
 * `npm run list-resources` - Lists all the schema-declared types and API versions referenced in [schemas/2019-04-01/deploymentTemplate.json](/schemas/2019-04-01/deploymentTemplate.json).
-* `npm run generate-all` - Re-generates all schemas for providers which have been whitelisted in [whitelist.ts](./whitelist.ts).
+* `npm run generate-all` - Re-generates all schemas for providers which have been added to [autogenlist.ts](./autogenlist.ts).
 * `npm run generate-single {basePath}` - Re-generates schemas for all resource types and API versions found under a given base path.
 * See [package.json](./package.json) for the full set of commands.
 
@@ -37,6 +37,6 @@ View the Pipeline Configuration [here](/azure-pipelines-autogen.yml).
 2. Run `npm run generate-single {basePath}` to generate schemas for them. Save the console output to file.
 3. Remove all references to the provider namespace from [schemas/2019-04-01/deploymentTemplate.json](/schemas/2019-04-01/deploymentTemplate.json).
 4. If there are any modifications to [schemas/common/autogeneratedResources.json](/schemas/common/autogeneratedResources.json), you may need to make corresponding changes to [schemas/2014-04-01-preview/deploymentTemplate.json](/schemas/2014-04-01-preview/deploymentTemplate.json) and [schemas/2015-01-01/deploymentTemplate.json](/schemas/2015-01-01/deploymentTemplate.json).
-5. Add the {basePath} to the whitelist array in [whitelist.ts](./whitelist.ts).
+5. Add the {basePath} to the autogenlist array in [autogenlist.ts](./autogenlist.ts).
 6. Commit all the changed files, and submit a PR to this repo with title "Onboarding {provider} for autogeneration". Include the full output saved in step 2 as a comment.
 7. Ensure all CI tests pass, and ask the team to review the PR (including tool output) to ensure that all expected resources have been detected and generated as expected.

generator/whitelist.ts → generator/autogenlist.ts

@@ -1,10 +1,10 @@
-import { ScopeType, WhitelistConfig } from './models';
+import { ScopeType, AutogenlistConfig } from './models';
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
 import { postProcessor as machineLearningPostProcessor } from './processors/Microsoft.MachineLearning';
 import { lowerCaseEquals } from './utils';
 
 // Run "npm run list-basepaths" to discover all the valid readme files to add to this list
-const whitelist: WhitelistConfig[] = [
+const autogenlist: AutogenlistConfig[] = [
     {
         basePath: 'addons/resource-manager',
         namespace: 'Microsoft.Addons',
@@ -37,10 +37,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'appconfiguration/resource-manager',
         namespace: 'Microsoft.AppConfiguration',
     },
-    {
+    /*{ Note(jcotillo) Temp exclusion due to an autorest issue.
         basePath: 'apimanagement/resource-manager',
         namespace: 'Microsoft.ApiManagement',
-    },
+    },*/
     {
         basePath: 'appplatform/resource-manager',
         namespace: 'Microsoft.AppPlatform',
@@ -73,6 +73,28 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'blockchain/resource-manager',
         namespace: 'Microsoft.Blockchain',
     },
+    {
+        basePath: 'blueprint/resource-manager',
+        namespace: 'Microsoft.Blueprint',
+        resourceConfig: [
+            {
+                type: 'blueprintAssignments',
+                scopes: ScopeType.Subcription | ScopeType.ManagementGroup,
+            },
+            {
+                type: 'blueprints',
+                scopes: ScopeType.Subcription | ScopeType.ManagementGroup,
+            },
+            {
+                type: 'blueprints/artifacts',
+                scopes: ScopeType.Subcription | ScopeType.ManagementGroup,
+            },
+            {
+                type: 'blueprints/versions',
+                scopes: ScopeType.Subcription | ScopeType.ManagementGroup,
+            },
+        ]
+    },
     {
         basePath: 'botservice/resource-manager',
         namespace: 'Microsoft.BotService',
@@ -81,6 +103,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'billing/resource-manager',
         namespace: 'Microsoft.Billing',
     },
+    {
+        basePath: 'changeanalysis/resource-manager',
+        namespace: 'Microsoft.ChangeAnalysis',
+    },
     {
         basePath: 'cognitiveservices/resource-manager',
         namespace: 'Microsoft.CognitiveServices',
@@ -169,6 +195,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'deploymentmanager/resource-manager',
         namespace: 'Microsoft.DeploymentManager',
     },
+    {
+        basePath: 'desktopvirtualization/resource-manager',
+        namespace: 'Microsoft.DesktopVirtualization',
+    },
     {
       basePath: 'digitaltwins/resource-manager',
       namespace: 'Microsoft.DigitalTwins',
@@ -216,6 +246,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'hdinsight/resource-manager',
         namespace: 'Microsoft.HDInsight',
     },
+    {
+        basePath: 'resourcehealth/resource-manager',
+        namespace: 'Microsoft.ResourceHealth',
+    }, 
     {
         basePath: 'EnterpriseKnowledgeGraph/resource-manager',
         namespace: 'Microsoft.EnterpriseKnowledgeGraph',
@@ -261,6 +295,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'iotspaces/resource-manager',
         namespace: 'Microsoft.IoTSpaces',
     },
+    {
+        basePath: 'intune/resource-manager',
+        namespace: 'Microsoft.Intune',
+    },
     {
         basePath: 'labservices/resource-manager',
         namespace: 'Microsoft.LabServices',
@@ -318,6 +356,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'mariadb/resource-manager',
         namespace: 'Microsoft.DBforMariaDB',
     },
+    {
+        basePath: 'marketplace/resource-manager',
+        namespace: 'Microsoft.Marketplace',
+    },
     {
         basePath: 'machinelearningservices/resource-manager',
         namespace: 'Microsoft.MachineLearningServices',
@@ -376,6 +418,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'powerbiembedded/resource-manager',
         namespace: 'Microsoft.PowerBI',
     },
+    {
+        basePath: 'redhatopenshift/resource-manager',
+        namespace: 'Microsoft.RedHatOpenShift',
+    },
     {
         basePath: 'resources/resource-manager',
         namespace: 'Microsoft.Resources',
@@ -391,13 +437,14 @@ const whitelist: WhitelistConfig[] = [
         ],
         postProcessor: resourcesPostProcessor,
     },
-    /* antmarti - temporarily commented out to unblock generation.
-                  this appears to have been broken by https://github.com/Azure/azure-rest-api-specs/pull/8903
     {
         basePath: 'relay/resource-manager',
         namespace: 'Microsoft.Relay',
     },
-    */
+    {
+        basePath: 'reservations/resource-manager',
+        namespace: 'Microsoft.Capacity',
+    },
     {
         basePath: 'servicebus/resource-manager',
         namespace: 'Microsoft.ServiceBus',
@@ -455,10 +502,6 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'vmwarecloudsimple/resource-manager',
         namespace: 'Microsoft.VMwareCloudSimple',
     },
-    {
-        basePath: 'vmwarevirtustream/resource-manager',
-        namespace: 'Microsoft.VMwareVirtustream',
-    },
     {
         basePath: 'visualstudio/resource-manager',
         namespace: 'Microsoft.VisualStudio',
@@ -488,6 +531,16 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'support/resource-manager',
         namespace: 'Microsoft.Support',
     },
+    {
+        basePath: 'softwareplan/resource-manager',
+        namespace: 'Microsoft.SoftwarePlan',
+        resourceConfig: [
+            {
+                type: 'hybridUseBenefits',
+                scopes: ScopeType.Extension,
+            },
+        ]
+    },
     {
         basePath: 'timeseriesinsights/resource-manager',
         namespace: 'Microsoft.TimeSeriesInsights',
@@ -496,6 +549,10 @@ const whitelist: WhitelistConfig[] = [
         basePath: 'imagebuilder/resource-manager',
         namespace: 'Microsoft.VirtualMachineImages',
     },
+    {
+        basePath: 'vmware/resource-manager',
+        namespace: 'Microsoft.AVS',
+    },
     {
         basePath: 'windowsesu/resource-manager',
         namespace: 'Microsoft.WindowsESU',
@@ -506,10 +563,10 @@ const whitelist: WhitelistConfig[] = [
     }
 ];
 
-export function getWhitelist(): WhitelistConfig[] {
-    return whitelist;
+export function getAutogenlist(): AutogenlistConfig[] {
+    return autogenlist;
 }
 
-export function findWhitelistEntries(basePath: string): WhitelistConfig[] {
-    return whitelist.filter(w => lowerCaseEquals(w.basePath, basePath));
+export function findAutogenEntries(basePath: string): AutogenlistConfig[] {
+    return autogenlist.filter(w => lowerCaseEquals(w.basePath, basePath));
 }

generator/cmd/generateall.ts

@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import { SchemaConfiguration, generateSchemas, clearAutogeneratedSchemaRefs, saveAutogeneratedSchemaRefs } from '../generate';
-import { getWhitelist } from '../whitelist';
+import { getAutogenlist } from '../autogenlist';
 import chalk from 'chalk';
 import { flatten, chunk } from 'lodash';
 import { executeSynchronous, chunker } from '../utils';
@@ -22,25 +22,25 @@ function parseParams(): GenerateAllParams {
 executeSynchronous(async () => {
     const params = parseParams();
 
-    let filteredWhitelist = getWhitelist();
+    let filteredAutogenlist = getAutogenlist();
     if (params.batchCount !== undefined && params.batchIndex !== undefined) {
-        filteredWhitelist = chunker(filteredWhitelist, params.batchCount)[params.batchIndex];
+        filteredAutogenlist = chunker(filteredAutogenlist, params.batchCount)[params.batchIndex];
     }
 
     await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
 
-    await clearAutogeneratedSchemaRefs(filteredWhitelist);
+    await clearAutogeneratedSchemaRefs(filteredAutogenlist);
 
     const schemaConfigs: SchemaConfiguration[] = [];
     const errors = [];
-    for (const whitelistConfig of filteredWhitelist) {
+    for (const autogenlistConfig of filteredAutogenlist) {
         try {
-            const readme = await validateAndReturnReadmePath(whitelistConfig.basePath);
+            const readme = await validateAndReturnReadmePath(autogenlistConfig.basePath);
 
-            const newConfigs = await generateSchemas(readme, whitelistConfig);
+            const newConfigs = await generateSchemas(readme, autogenlistConfig);
             schemaConfigs.push(...newConfigs);
         } catch(error) {
-            console.log(chalk.red(`Caught exception processing whitelist entry ${whitelistConfig.basePath}.`));
+            console.log(chalk.red(`Caught exception processing autogenlist entry ${autogenlistConfig.basePath}.`));
             console.log(chalk.red(error));
 
             errors.push(error);

generator/cmd/generatesingle.ts

@@ -2,7 +2,7 @@ import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import { generateSchemas, saveAutogeneratedSchemaRefs } from '../generate';
 import process from 'process';
-import { findWhitelistEntries } from '../whitelist';
+import { findAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
 import { executeSynchronous } from '../utils';
 
@@ -18,17 +18,17 @@ executeSynchronous(async () => {
     }
 
     const schemaConfigs = [];
-    const whitelistEntries = findWhitelistEntries(basePath);
+    const autogenEntries = findAutogenEntries(basePath);
 
-    if (whitelistEntries.length === 0) {
+    if (autogenEntries.length === 0) {
         const localSchemaConfigs = await generateSchemas(readme);
         schemaConfigs.push(...localSchemaConfigs);
     } else {
-        for (const whitelistConfig of whitelistEntries) {
-            console.log(`Using whitelist config:`)
-            console.log(chalk.green(JSON.stringify(whitelistConfig, null, 2)));
+        for (const autogenlistConfig of autogenEntries) {
+            console.log(`Using autogenlist config:`)
+            console.log(chalk.green(JSON.stringify(autogenlistConfig, null, 2)));
 
-            const localSchemaConfigs = await generateSchemas(readme, whitelistConfig);
+            const localSchemaConfigs = await generateSchemas(readme, autogenlistConfig);
             schemaConfigs.push(...localSchemaConfigs);
         }
     }

generator/cmd/listbasepaths.ts

@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import chalk from 'chalk';
-import { findWhitelistEntries } from '../whitelist';
+import { findAutogenEntries } from '../autogenlist';
 import { executeSynchronous, lowerCaseEquals } from '../utils';
 import { getApiVersionsByNamespace } from '../generate';
 import { keys, partition } from 'lodash';
@@ -12,18 +12,18 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         const readme = await validateAndReturnReadmePath(basePath);
         const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const whitelistEntries = findWhitelistEntries(basePath);
+        const autogenlistEntries = findAutogenEntries(basePath);
 
-        const [whitelisted, unwhitelisted] = partition(
+        const [autogened, unautogened] = partition(
             namespaces,
-            n => whitelistEntries.findIndex(w => lowerCaseEquals(w.namespace, n)) > -1);
+            n => autogenlistEntries.findIndex(w => lowerCaseEquals(w.namespace, n)) > -1);
 
-        if (unwhitelisted.length === 0) {
-            console.log(`Discovered '${chalk.green(basePath)}'. Whitelisted for auto-generation: ${chalk.green('yes')}.`);
-        } else if (whitelisted.length > 0) {
-            console.log(`Discovered '${chalk.green(basePath)}'. Whitelisted for auto-generation: ${chalk.yellow('partial')}. Missing: ${unwhitelisted.map(p => chalk.yellow(p)).join(', ')}.`);
+        if (unautogened.length === 0) {
+            console.log(`Discovered '${chalk.green(basePath)}'. autogened for auto-generation: ${chalk.green('yes')}.`);
+        } else if (autogened.length > 0) {
+            console.log(`Discovered '${chalk.green(basePath)}'. autogened for auto-generation: ${chalk.yellow('partial')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`);
         } else {
-            console.log(`Discovered '${chalk.green(basePath)}'. Whitelisted for auto-generation: ${chalk.red('no')}. Missing: ${unwhitelisted.map(p => chalk.yellow(p)).join(', ')}.`);
+            console.log(`Discovered '${chalk.green(basePath)}'. autogened for auto-generation: ${chalk.red('no')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`);
         }
     }
 });

generator/constants.ts

@@ -29,11 +29,11 @@ export const managementGroupRootSchema = {
 };
 export const generatedSchemasTemplatePath = path.join(__dirname, 'resources/autogeneratedResources_template.json');
 
-export const autorestCoreVersion = '3.0.6257';
+export const autorestCoreVersion = '3.0.6274';
 export const azureresourceschemaVersion = '3.0.79';
 
 // paths in this list won't even appear in list-basepaths
-export const blacklist = [
+export const blocklist = [
     /* Azure Stack resource providers */
     'azsadmin/resource-manager/azurebridge',
     'azsadmin/resource-manager/backup',