Skip to content

Commit

Permalink
Remove the need for cmd/pulumi-gen-pulumiservice (#240)
Browse files Browse the repository at this point in the history
We don't need to maintain our own gen code, we can use the pulumi binary
for that. This makes it important that we use the right version of the
pulumi binary, so we use the Makefile to ensure the version of pulumi
used matches the version of pulumi we depend on in `provider/go.mod`.

We use a similar pattern in other native providers.

I'm tagging @pierskarsenbarg and @komalali since they were the suggested
reviewers. If I should tag someone else here, please let me know.
  • Loading branch information
iwahbe authored Apr 6, 2024
1 parent 07693e0 commit d58439e
Show file tree
Hide file tree
Showing 131 changed files with 315 additions and 2,769 deletions.
105 changes: 105 additions & 0 deletions .gradle/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// *** WARNING: this file was generated. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***
plugins {
id("java-library")
id("maven-publish")
}

var gprUser = project.findProperty("gpr.user") ?: System.getenv("GPR_USER")
var gprToken = project.findProperty("gpr.token") ?: System.getenv("GPR_TOKEN")

java {
toolchain {
languageVersion = JavaLanguageVersion.of(11)
}
}

compileJava {
options.fork = true
options.forkOptions.jvmArgs.addAll(["-Xmx4g"])
}

repositories {
maven { // The google mirror is less flaky than mavenCentral()
url("https://maven-central.storage-download.googleapis.com/maven2/")
}
mavenCentral()
mavenLocal()
if (gprUser) {
repositories {
maven {
url = uri("https://maven.pkg.github.com/pulumi/pulumi-java")
credentials {
username = gprUser
password = gprToken
}
}
}
}
}

var pulumiJavaSdkVersion = System.getenv("PULUMI_JAVA_SDK_VERSION") ?: "0.0.1"

dependencies {
implementation "com.pulumi:pulumi:$pulumiJavaSdkVersion"
implementation("com.google.code.findbugs:jsr305:3.0.2")
api("com.google.guava:guava:30.1-jre") // FIXME: do we really want to expose this dep?
api("com.google.code.gson:gson:2.8.6") // make sure we don't clash with grpc deps

implementation("com.google.protobuf:protobuf-java:3.12.0") // make sure we don't clash with grpc deps
implementation("com.google.protobuf:protobuf-java-util:3.12.0") // make sure we don't clash with grpc deps

def junitVersion = "5.7.2"
testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
testImplementation("org.junit.jupiter:junit-jupiter-params:${junitVersion}")
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
testImplementation("org.assertj:assertj-core:3.20.2")
testImplementation("org.mockito:mockito-core:3.12.4")
}

test {
useJUnitPlatform()
testLogging {
showStandardStreams = true
exceptionFormat = 'full'

// set options for log level LIFECYCLE
events = ['failed']

info {
events = ['failed', 'skipped']
}

debug {
events = ['started', 'skipped', 'failed']
}
}
}

publishing {
publications {
mavenJava(MavenPublication) {
groupId = 'com.pulumi'
artifactId = 'pulumiservice'
version = System.getenv("SDK_VERSION") ?: "0.0.0-unset"
from components.java
}
}
// from https://docs.github.com/en/actions/publishing-packages/publishing-java-packages-with-gradle
repositories {
maven {
name = "GitHubPackages"
url = "https://maven.pkg.github.com/pulumi/pulumi-pulumiservice"
credentials {
username = System.getenv("GITHUB_ACTOR")
password = System.getenv("GITHUB_TOKEN")
}
}
}
}

javadoc {
if (JavaVersion.current().isJava9Compatible()) {
options.addBooleanOption('html5', true)
}
}
File renamed without changes.
File renamed without changes.
File renamed without changes.
14 changes: 14 additions & 0 deletions .gradle/settings.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// *** WARNING: this file was generated. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***

pluginManagement {
repositories {
maven { // The google mirror is less flaky than mavenCentral()
url("https://maven-central.storage-download.googleapis.com/maven2/")
}
gradlePluginPortal()
}
}

rootProject.name = "com.pulumi.pulumiservice"
include("lib")
71 changes: 50 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ NODE_MODULE_NAME := @pulumi/pulumi-service
NUGET_PKG_NAME := Pulumi.PulumiService

PROVIDER := pulumi-resource-${PACK}
CODEGEN := pulumi-gen-${PACK}
VERSION ?= $(shell pulumictl get version)
PROVIDER_PATH := provider
VERSION_PATH := ${PROVIDER_PATH}/pkg/version.Version
Expand All @@ -18,13 +17,21 @@ GOPATH := $(shell go env GOPATH)
WORKING_DIR := $(shell pwd)
TESTPARALLELISM := 4

# The pulumi binary to use during generation
PULUMI := .pulumi/bin/pulumi

export PULUMI_IGNORE_AMBIENT_PLUGINS = true

ensure::
cd provider && go mod tidy
cd sdk && go mod tidy
cd examples && go mod tidy

gen::
(cd provider && go build -o $(WORKING_DIR)/bin/${CODEGEN} -ldflags "-X ${PROJECT}/${VERSION_PATH}=${VERSION}" ${PROJECT}/${PROVIDER_PATH}/cmd/$(CODEGEN))

build_sdks: dotnet_sdk go_sdk nodejs_sdk python_sdk java_sdk

gen_sdk_prerequisites: $(PULUMI)

provider::
(cd provider && VERSION=${VERSION} go generate cmd/${PROVIDER}/main.go)
Expand All @@ -36,32 +43,32 @@ provider_debug::
test_provider::
cd provider/pkg && go test -short -v -count=1 -cover -timeout 2h -parallel ${TESTPARALLELISM} ./...

dotnet_sdk:: DOTNET_VERSION := $(shell pulumictl get version --language dotnet)
dotnet_sdk::
dotnet_sdk: DOTNET_VERSION := $(shell pulumictl get version --language dotnet)
dotnet_sdk: gen_sdk_prerequisites
rm -rf sdk/dotnet
$(WORKING_DIR)/bin/$(CODEGEN) -version=${DOTNET_VERSION} dotnet $(SCHEMA_FILE) $(CURDIR)
$(PULUMI) package gen-sdk $(SCHEMA_FILE) --language dotnet
cd ${PACKDIR}/dotnet/&& \
echo "${DOTNET_VERSION}" >version.txt && \
dotnet build /p:Version=${DOTNET_VERSION}

go_sdk::
go_sdk: gen_sdk_prerequisites
rm -rf sdk/go
$(WORKING_DIR)/bin/$(CODEGEN) -version=${VERSION} go $(SCHEMA_FILE) $(CURDIR)
$(PULUMI) package gen-sdk $(SCHEMA_FILE) --language go

nodejs_sdk:: VERSION := $(shell pulumictl get version --language javascript)
nodejs_sdk::
nodejs_sdk: VERSION := $(shell pulumictl get version --language javascript)
nodejs_sdk: gen_sdk_prerequisites
rm -rf sdk/nodejs
$(WORKING_DIR)/bin/$(CODEGEN) -version=${VERSION} nodejs $(SCHEMA_FILE) $(CURDIR)
$(PULUMI) package gen-sdk $(SCHEMA_FILE) --language nodejs
cd ${PACKDIR}/nodejs/ && \
yarn install && \
yarn run tsc && \
cp ../../README.md ../../LICENSE package.json yarn.lock ./bin/ && \
sed -i.bak -e 's/\$${VERSION}/$(VERSION)/g' ./bin/package.json

python_sdk:: PYPI_VERSION := $(shell pulumictl get version --language python)
python_sdk::
python_sdk: PYPI_VERSION := $(shell pulumictl get version --language python)
python_sdk: gen_sdk_prerequisites
rm -rf sdk/python
$(WORKING_DIR)/bin/$(CODEGEN) -version=${VERSION} python $(SCHEMA_FILE) $(CURDIR)
$(PULUMI) package gen-sdk $(SCHEMA_FILE) --language python
cp README.md ${PACKDIR}/python/
cd ${PACKDIR}/python/ && \
python3 setup.py clean --all 2>/dev/null && \
Expand All @@ -70,16 +77,20 @@ python_sdk::
rm ./bin/setup.py.bak && \
cd ./bin && python3 setup.py build sdist

java_sdk:: RESOURCE_FOLDER := src/main/resources/com/pulumi/pulumiservice
java_sdk::
GRADLE_DIR := $(WORKING_DIR)/.gradle
GRADLE := $(GRADLE_DIR)/gradlew
java_sdk: RESOURCE_FOLDER := src/main/resources/com/pulumi/pulumiservice
java_sdk: gen_sdk_prerequisites
rm -rf sdk/java/{.gradle,build,src}
$(WORKING_DIR)/bin/$(CODEGEN) -version=${VERSION} java $(SCHEMA_FILE) $(CURDIR)
$(PULUMI) package gen-sdk $(SCHEMA_FILE) --language java
cp $(GRADLE_DIR)/settings.gradle sdk/java/settings.gradle
cp $(GRADLE_DIR)/build.gradle sdk/java/build.gradle
cd sdk/java && \
mkdir -p $(RESOURCE_FOLDER) && \
mkdir -p $(RESOURCE_FOLDER) && \
echo "$(VERSION)" > $(RESOURCE_FOLDER)/version.txt && \
echo '{"resource": true,"name": "pulumiservice","version": "$(VERSION)"}' > $(RESOURCE_FOLDER)/plugin.json && \
PULUMI_JAVA_SDK_VERSION=0.10.0 ./gradlew --console=plain build && \
PULUMI_JAVA_SDK_VERSION=0.10.0 ./gradlew --console=plain publishToMavenLocal
PULUMI_JAVA_SDK_VERSION=0.10.0 $(GRADLE) --console=plain build && \
PULUMI_JAVA_SDK_VERSION=0.10.0 $(GRADLE) --console=plain publishToMavenLocal

.PHONY: build
build:: gen provider dotnet_sdk go_sdk nodejs_sdk python_sdk java_sdk
Expand All @@ -96,7 +107,7 @@ lint::
install:: install_nodejs_sdk install_dotnet_sdk
cp $(WORKING_DIR)/bin/${PROVIDER} ${GOPATH}/bin

GO_TEST := go test -v -count=1 -cover -timeout 2h -parallel ${TESTPARALLELISM}
GO_TEST := go test -v -count=1 -cover -timeout 2h -parallel ${TESTPARALLELISM}

install_dotnet_sdk::
rm -rf $(WORKING_DIR)/nuget/$(NUGET_PKG_NAME).*.nupkg
Expand All @@ -114,4 +125,22 @@ install_nodejs_sdk::
yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin

install_java_sdk::
cd sdk/java && ./gradlew publishToMavenLocal
cd sdk/java && $(GRADLE) publishToMavenLocal


# Keep the version of the pulumi binary used for code generation in sync with the version
# of the dependency used by github.com/pulumi/pulumi-pulumiservice/provider

$(PULUMI): HOME := $(WORKING_DIR)
$(PULUMI): provider/go.mod
@ PULUMI_VERSION="$$(cd provider && go list -m github.com/pulumi/pulumi/pkg/v3 | awk '{print $$2}')"; \
if [ -x $(PULUMI) ]; then \
CURRENT_VERSION="$$($(PULUMI) version)"; \
if [ "$${CURRENT_VERSION}" != "$${PULUMI_VERSION}" ]; then \
echo "Upgrading $(PULUMI) from $${CURRENT_VERSION} to $${PULUMI_VERSION}"; \
rm $(PULUMI); \
fi; \
fi; \
if ! [ -x $(PULUMI) ]; then \
curl -fsSL https://get.pulumi.com | sh -s -- --version "$${PULUMI_VERSION#v}"; \
fi
47 changes: 27 additions & 20 deletions examples/examples_yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import (
"io"
"os"
"path"
"path/filepath"
"strings"
"testing"

"github.com/google/uuid"
"github.com/pulumi/pulumi/pkg/v3/engine"
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -126,36 +128,46 @@ func TestYamlTeamsExample(t *testing.T) {

func TestYamlStackTagsExample(t *testing.T) {

// Set up tmpdir with a Pulumi.yml with no resources
// mimicking the deletion of resource
newProgram := YamlProgram{
// yaml-stack-tags-example applies tags to it's own stack. To do this, we need to
// first create an empty stack, then add the stack tag.

tmpdir := writePulumiYaml(t, YamlProgram{
Name: "yaml-stack-tags-example",
Runtime: "yaml",
Description: "A minimal Pulumi YAML program",
}
})

tmpdir := writePulumiYaml(t, newProgram)

cwd, _ := os.Getwd()
cwd, err := os.Getwd()
require.NoError(t, err)

integration.ProgramTest(t, &integration.ProgramTestOptions{
Quick: true,
SkipRefresh: true,
Dir: path.Join(cwd, ".", "yaml-stack-tags"),
StackName: "test-stack",
PrepareProject: func(_ *engine.Projinfo) error {
return nil
},
Dir: tmpdir,
EditDirs: []integration.EditDir{
{
Dir: tmpdir,
Dir: path.Join(cwd, ".", "yaml-stack-tags"),
},
// Reapply the same thing again, except this time we expect there to be no changes
{
Dir: tmpdir,
Dir: path.Join(cwd, ".", "yaml-stack-tags"),
ExpectNoChanges: true,
},
},

// Setting Config and PrepareProject works around the bug introduced in
// https://github.com/pulumi/pulumi/pull/14695:
//
// - Setting a config value ensures that the stack file is created.
//
// - PrepareProject is used to force the bookkeeping file to be created.
//
// Once https://github.com/pulumi/pulumi/pull/15863 merges, both fields
// can be removed.
Config: map[string]string{"ensure-config": "true"},
PrepareProject: func(info *engine.Projinfo) error {
return os.MkdirAll(filepath.Join(info.Root, ".pulumi"), 0700)
},
})
}

Expand Down Expand Up @@ -220,12 +232,7 @@ func TestYamlTeamStackPermissionsExample(t *testing.T) {
cwd, _ := os.Getwd()
integration.ProgramTest(t, &integration.ProgramTestOptions{
Quick: true,
// Name is specified in yaml-team-stack-permissions/Pulumi.yaml, so this has to be consistent
StackName: "dev",
Dir: path.Join(cwd, ".", "yaml-team-stack-permissions"),
PrepareProject: func(_ *engine.Projinfo) error {
return nil
},
Dir: path.Join(cwd, ".", "yaml-team-stack-permissions"),
})
}

Expand Down
Empty file removed examples/yaml-stack-tags/.pulumi
Empty file.
Empty file.
4 changes: 2 additions & 2 deletions examples/yaml-stack-tags/Pulumi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ resources:
type: pulumiservice:StackTag
properties:
organization: service-provider-test-org
project: yaml-stack-tags-example
stack: test-stack
project: ${pulumi.project}
stack: ${pulumi.stack}
name: testing-tag
value: bar
2 changes: 1 addition & 1 deletion examples/yaml-team-stack-permissions/Pulumi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ resources:
properties:
organization: ${team.organizationName}
project: yaml-team-stack-example
stack: dev
stack: ${pulumi.stack}
team: ${team.name}
# edit access
permission: 102
Loading

0 comments on commit d58439e

Please sign in to comment.