Skip to content

Commit

Permalink
examples: add gaussian defense and ability to use mappings to client
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtsextonMITRE
jtsextonMITRE committed Sep 18, 2024
1 parent 81ac8b2 commit 91c5b4e
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 150 deletions.
204 changes: 55 additions & 149 deletions examples/mnist-classifier-demo/demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
},
{
"cell_type": "code",
"execution_count": 80,
"execution_count": null,
"metadata": {
"tags": []
},
Expand All @@ -53,12 +53,13 @@
},
{
"cell_type": "code",
"execution_count": 81,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Import packages from the Python standard library\n",
"import importlib.util\n",
"import json\n",
"import os\n",
"import sys\n",
"import pprint\n",
Expand Down Expand Up @@ -176,7 +177,7 @@
},
{
"cell_type": "code",
"execution_count": 131,
"execution_count": null,
"metadata": {
"tags": []
},
Expand All @@ -194,28 +195,9 @@
},
{
"cell_type": "code",
"execution_count": 132,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[2m2024-09-18 12:52:25\u001b[0m [\u001b[31m\u001b[1merror \u001b[0m] \u001b[1mError code 400 returned. \u001b[0m \u001b[36mdata\u001b[0m=\u001b[35m{'username': 'pluginuser', 'email': '[email protected]', 'password': 'pleasemakesuretoPLUGINthecomputer', 'confirmPassword': 'pleasemakesuretoPLUGINthecomputer'}\u001b[0m \u001b[36mmethod\u001b[0m=\u001b[35mPOST\u001b[0m \u001b[36mresponse\u001b[0m=\u001b[35m{\"message\": \"Bad Request - The username on the registration form is not available. Please select another and resubmit.\"}\n",
"\u001b[0m \u001b[36murl\u001b[0m=\u001b[35mhttp://localhost:20080/api/v1/users/\u001b[0m\n"
]
},
{
"data": {
"text/plain": [
"{'username': 'pluginuser', 'status': 'Login successful'}"
]
},
"execution_count": 132,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"try:\n",
" client.users.create('pluginuser','[email protected]','pleasemakesuretoPLUGINthecomputer','pleasemakesuretoPLUGINthecomputer')\n",
Expand All @@ -233,7 +215,7 @@
},
{
"cell_type": "code",
"execution_count": 154,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -261,7 +243,7 @@
},
{
"cell_type": "code",
"execution_count": 134,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -283,7 +265,7 @@
},
{
"cell_type": "code",
"execution_count": 135,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -312,19 +294,9 @@
},
{
"cell_type": "code",
"execution_count": 137,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting fgm job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"job_time_limit = '1h'\n",
"\n",
Expand All @@ -350,7 +322,7 @@
},
{
"cell_type": "code",
"execution_count": 149,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -366,15 +338,15 @@
" {\"job_id\": str(prev_job['id']),\n",
" \"tar_name\": tn,\n",
" \"data_dir\": dd,\n",
" \"model_name\": MODEL_NAME, \"model_version\": str(-1)},\n",
" \"model_name\": MODEL_NAME, \"model_version\": str(-1)}, # -1 means get the latest\n",
" job_time_limit\n",
" )\n",
" return infer_job"
]
},
{
"cell_type": "code",
"execution_count": 150,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -391,26 +363,16 @@
},
{
"cell_type": "code",
"execution_count": 140,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting infer job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"infer_fgm = infer(experiment_id, queue_id, infer_ep, fgm_job, defense=False)"
]
},
{
"cell_type": "code",
"execution_count": 141,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -419,19 +381,11 @@
},
{
"cell_type": "code",
"execution_count": 142,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting defense job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"job_time_limit = '1h'\n",
"wait_for_job(fgm_job, 'defense')\n",
Expand All @@ -440,45 +394,29 @@
" f\"defense job for {experiment_id}\",\n",
" queue_id,\n",
" defense_ep,\n",
" {\"job_id\": str(fgm_job['id']),\"def_type\":\"spatial_smoothing\"}, # -1 means get the latest\n",
" {\n",
" \"job_id\": str(fgm_job['id']),\n",
" \"def_type\":\"spatial_smoothing\",\n",
" \"defense_kwargs\": json.dumps({})\n",
" }, \n",
" job_time_limit\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 143,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting infer job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"infer_spatial = infer(experiment_id, queue_id, infer_ep, spatial_job, defense=True)"
]
},
{
"cell_type": "code",
"execution_count": 144,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting defense job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"job_time_limit = '1h'\n",
"wait_for_job(fgm_job, 'defense')\n",
Expand All @@ -489,104 +427,72 @@
" defense_ep,\n",
" {\n",
" \"job_id\": str(fgm_job['id']),\n",
" \"def_type\":\"jpeg_compression\"\n",
" }, # -1 means get the latest\n",
" \"def_type\":\"jpeg_compression\",\n",
" \"defense_kwargs\": json.dumps({})\n",
" },\n",
" job_time_limit\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 145,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting infer job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"infer_jpeg = infer(experiment_id, queue_id, infer_ep, jpeg_comp_job, defense=True)"
]
},
{
"cell_type": "code",
"execution_count": 156,
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting defense job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"job_time_limit = '1h'\n",
"wait_for_job(fgm_job, 'defense')\n",
"jpeg_comp_job = client.experiments.create_jobs_by_experiment_id(\n",
"gaussian_job = client.experiments.create_jobs_by_experiment_id(\n",
" experiment_id,\n",
" f\"defense job for {experiment_id}\",\n",
" queue_id,\n",
" defense_ep,\n",
" {\n",
" \"job_id\": str(fgm_job['id']),\n",
" \"def_type\":\"gaussian_augmentation\"\n",
" }, # -1 means get the latest\n",
" \"def_type\":\"gaussian_augmentation\",\n",
" \"defense_kwargs\": json.dumps({\n",
" \"augmentation\": False,\n",
" \"ratio\": 1,\n",
" \"sigma\": 1,\n",
" \"apply_fit\": False,\n",
" \"apply_predict\": True\n",
" })\n",
" }, \n",
" job_time_limit\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 145,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Job finished. Starting infer job.'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"infer_jpeg = infer(experiment_id, queue_id, infer_ep, jpeg_comp_job, defense=True)"
"infer_gaussian = infer(experiment_id, queue_id, infer_ep, gaussian_job, defense=True)"
]
},
{
"cell_type": "code",
"execution_count": 155,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'training_time_in_minutes': 0.32976753333333336, 'accuracy': 0.9775166511535645, 'auc': 0.9987682700157166, 'loss': 0.07407279312610626, 'precision': 0.9809511303901672, 'recall': 0.9750000238418579}\n",
"{'accuracy': 0.11217948794364929, 'auc': 0.6169368028640747, 'precision': 0.09878776967525482, 'loss': 3.25475811958313, 'recall': 0.0546875}\n",
"{'accuracy': 0.11548477411270142, 'auc': 0.6298573613166809, 'loss': 3.010637044906616, 'precision': 0.10013880580663681, 'recall': 0.05058092996478081}\n",
"{'auc': 0.617414653301239, 'precision': 0.12656284868717194, 'accuracy': 0.1341145783662796, 'loss': 2.9532642364501953, 'recall': 0.05779246613383293}\n"
]
}
],
"outputs": [],
"source": [
"print(get_metrics(training_job))\n",
"print(get_metrics(infer_fgm))\n",
"print(get_metrics(infer_jpeg))\n",
"print(get_metrics(infer_spatial))"
"print(get_metrics(infer_spatial))\n",
"print(get_metrics(infer_gaussian))"
]
}
],
Expand Down
2 changes: 2 additions & 0 deletions examples/mnist-classifier-demo/src/defense.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ parameters:
norm: "inf"
seed: -1
def_type: spatial_smoothing
defense_kwargs: {}

tasks:
load_artifacts_for_job:
Expand Down Expand Up @@ -182,6 +183,7 @@ graph:
image_size: $image_size
batch_size: $batch_size
def_type: $def_type
defense_kwargs: $defense_kwargs
distance_metrics:
- name: l_infinity_norm
func: l_inf_norm
Expand Down
Loading

0 comments on commit 91c5b4e

Please sign in to comment.