Loss of keypoints or joints wasn't stable

[Tommy-Hsu]

Dataset: Posetrack2018
Pretrained Backbone: MMPose HRNet (heatmap-based) mAP in PCKh is 84.5
Classifier loss weight (same as original) : token_loss=1.0, joint_loss=1.0
Tokenizer loss weight (same as original) : joint_loss_w=1.0, e_loss_w=15.0,

In both stage I and stage II, the joints (keypoints) loss couldn't be stable.

In stage II, although the mAP could reach value 78.9, but the kpt_loss still get jitter. And stuck in 78.8 for a long time.

{"mode": "train", "epoch": 51, "iter": 100, "lr": 0.00994, "memory": 8152, "data_time": 0.23968, "token_loss": 4.51357, "kpt_loss": 27.17966, "top1-acc": 7.53849, "top2-acc": 13.75345, "top5-acc": 27.79423, "loss": 31.69323, "time": 2.27464}
{"mode": "train", "epoch": 51, "iter": 200, "lr": 0.00994, "memory": 8152, "data_time": 0.00046, "token_loss": 4.48997, "kpt_loss": 33.69612, "top1-acc": 7.64258, "top2-acc": 13.93635, "top5-acc": 27.98081, "loss": 38.18609, "time": 2.10606}
{"mode": "train", "epoch": 51, "iter": 300, "lr": 0.00994, "memory": 8152, "data_time": 0.00047, "token_loss": 4.49782, "kpt_loss": 41.50038, "top1-acc": 7.62052, "top2-acc": 13.87718, "top5-acc": 27.95715, "loss": 45.9982, "time": 2.11689}
{"mode": "train", "epoch": 52, "iter": 100, "lr": 0.00994, "memory": 8152, "data_time": 0.2526, "token_loss": 4.49968, "kpt_loss": 34.61265, "top1-acc": 7.65751, "top2-acc": 13.92073, "top5-acc": 27.94899, "loss": 39.11233, "time": 2.33765}
{"mode": "train", "epoch": 52, "iter": 200, "lr": 0.00994, "memory": 8152, "data_time": 0.00047, "token_loss": 4.49681, "kpt_loss": 34.67456, "top1-acc": 7.60719, "top2-acc": 13.86776, "top5-acc": 28.00195, "loss": 39.17138, "time": 2.11962}
{"mode": "train", "epoch": 52, "iter": 300, "lr": 0.00994, "memory": 8152, "data_time": 0.00052, "token_loss": 4.48208, "kpt_loss": 23.88715, "top1-acc": 7.73277, "top2-acc": 14.07238, "top5-acc": 28.23633, "loss": 28.36923, "time": 2.12533}
{"mode": "train", "epoch": 53, "iter": 100, "lr": 0.00994, "memory": 8152, "data_time": 0.25716, "token_loss": 4.49984, "kpt_loss": 23.98815, "top1-acc": 7.66521, "top2-acc": 13.94566, "top5-acc": 28.06239, "loss": 28.48799, "time": 2.34329}
{"mode": "train", "epoch": 53, "iter": 200, "lr": 0.00994, "memory": 8152, "data_time": 0.00055, "token_loss": 4.49879, "kpt_loss": 35.7736, "top1-acc": 7.58973, "top2-acc": 13.83686, "top5-acc": 27.86868, "loss": 40.2724, "time": 2.11937}
{"mode": "train", "epoch": 53, "iter": 300, "lr": 0.00994, "memory": 8152, "data_time": 0.00046, "token_loss": 4.4779, "kpt_loss": 47.95477, "top1-acc": 7.6883, "top2-acc": 14.08123, "top5-acc": 28.2801, "loss": 52.43267, "time": 2.11913}
{"mode": "train", "epoch": 54, "iter": 100, "lr": 0.00993, "memory": 8152, "data_time": 0.25539, "token_loss": 4.49527, "kpt_loss": 16.89038, "top1-acc": 7.63304, "top2-acc": 13.92624, "top5-acc": 27.973, "loss": 21.38565, "time": 2.34093}
{"mode": "train", "epoch": 54, "iter": 200, "lr": 0.00993, "memory": 8152, "data_time": 0.00046, "token_loss": 4.48521, "kpt_loss": 23.21045, "top1-acc": 7.66441, "top2-acc": 13.9938, "top5-acc": 28.18807, "loss": 27.69566, "time": 2.1227}
{"mode": "train", "epoch": 54, "iter": 300, "lr": 0.00993, "memory": 8152, "data_time": 0.00043, "token_loss": 4.48356, "kpt_loss": 57.89904, "top1-acc": 7.73265, "top2-acc": 14.07284, "top5-acc": 28.21151, "loss": 62.3826, "time": 2.1202}
{"mode": "train", "epoch": 55, "iter": 100, "lr": 0.00993, "memory": 8152, "data_time": 0.25444, "token_loss": 4.48711, "kpt_loss": 26.28602, "top1-acc": 7.72484, "top2-acc": 14.03631, "top5-acc": 28.21197, "loss": 30.77314, "time": 2.33846}
{"mode": "train", "epoch": 55, "iter": 200, "lr": 0.00993, "memory": 8152, "data_time": 0.00048, "token_loss": 4.48039, "kpt_loss": 40.37974, "top1-acc": 7.77275, "top2-acc": 14.10489, "top5-acc": 28.29331, "loss": 44.86014, "time": 2.11609}
{"mode": "train", "epoch": 55, "iter": 300, "lr": 0.00993, "memory": 8152, "data_time": 0.00045, "token_loss": 4.49012, "kpt_loss": 26.97002, "top1-acc": 7.68394, "top2-acc": 14.00563, "top5-acc": 28.08605, "loss": 31.46014, "time": 2.11909}
{"mode": "val", "epoch": 55, "iter": 176, "lr": 0.00993, "Head AP": 86.05974, "Shou AP": 86.50651, "Elb AP": 74.43714, "Wri AP": 67.21827, "Hip AP": 79.09285, "Knee AP": 78.06248, "Ankl AP": 72.67446, "Total AP": 78.27751}
{"mode": "train", "epoch": 56, "iter": 100, "lr": 0.00993, "memory": 8152, "data_time": 0.2397, "token_loss": 4.48571, "kpt_loss": 24.73939, "top1-acc": 7.69393, "top2-acc": 13.98116, "top5-acc": 28.13959, "loss": 29.2251, "time": 2.27062}
{"mode": "train", "epoch": 56, "iter": 200, "lr": 0.00993, "memory": 8152, "data_time": 0.00045, "token_loss": 4.48817, "kpt_loss": 20.96621, "top1-acc": 7.70232, "top2-acc": 14.01827, "top5-acc": 28.09869, "loss": 25.45438, "time": 2.10196}
{"mode": "train", "epoch": 56, "iter": 300, "lr": 0.00993, "memory": 8152, "data_time": 0.00048, "token_loss": 4.48009, "kpt_loss": 20.16352, "top1-acc": 7.73794, "top2-acc": 14.0918, "top5-acc": 28.19003, "loss": 24.64361, "time": 2.11093}
{"mode": "train", "epoch": 57, "iter": 100, "lr": 0.00993, "memory": 8152, "data_time": 0.2552, "token_loss": 4.47998, "kpt_loss": 28.74855, "top1-acc": 7.80147, "top2-acc": 14.09145, "top5-acc": 28.26505, "loss": 33.22854, "time": 2.34022}
{"mode": "train", "epoch": 57, "iter": 200, "lr": 0.00993, "memory": 8152, "data_time": 0.00051, "token_loss": 4.48006, "kpt_loss": 24.64193, "top1-acc": 7.75126, "top2-acc": 14.11546, "top5-acc": 28.3048, "loss": 29.12199, "time": 2.1152}
{"mode": "train", "epoch": 57, "iter": 300, "lr": 0.00993, "memory": 8152, "data_time": 0.00047, "token_loss": 4.47432, "kpt_loss": 32.68562, "top1-acc": 7.84915, "top2-acc": 14.13546, "top5-acc": 28.34145, "loss": 37.15994, "time": 2.11562}
{"mode": "train", "epoch": 58, "iter": 100, "lr": 0.00992, "memory": 8152, "data_time": 0.25489, "token_loss": 4.48551, "kpt_loss": 34.80063, "top1-acc": 7.79435, "top2-acc": 14.1497, "top5-acc": 28.27114, "loss": 39.28614, "time": 2.33564}
{"mode": "train", "epoch": 58, "iter": 200, "lr": 0.00992, "memory": 8152, "data_time": 0.00046, "token_loss": 4.45394, "kpt_loss": 23.34209, "top1-acc": 7.81939, "top2-acc": 14.19715, "top5-acc": 28.40223, "loss": 27.79603, "time": 2.11467}
{"mode": "train", "epoch": 58, "iter": 300, "lr": 0.00992, "memory": 8152, "data_time": 0.00047, "token_loss": 4.48717, "kpt_loss": 37.73394, "top1-acc": 7.7523, "top2-acc": 14.108, "top5-acc": 28.21772, "loss": 42.22111, "time": 2.11274}
{"mode": "train", "epoch": 59, "iter": 100, "lr": 0.00992, "memory": 8152, "data_time": 0.25087, "token_loss": 4.48151, "kpt_loss": 28.24467, "top1-acc": 7.777, "top2-acc": 14.1458, "top5-acc": 28.32112, "loss": 32.72618, "time": 2.33956}
{"mode": "train", "epoch": 59, "iter": 200, "lr": 0.00992, "memory": 8152, "data_time": 0.00046, "token_loss": 4.46839, "kpt_loss": 24.71504, "top1-acc": 7.8071, "top2-acc": 14.2508, "top5-acc": 28.41245, "loss": 29.18343, "time": 2.11834}
{"mode": "train", "epoch": 59, "iter": 300, "lr": 0.00992, "memory": 8152, "data_time": 0.00047, "token_loss": 4.46673, "kpt_loss": 30.30775, "top1-acc": 7.80722, "top2-acc": 14.19945, "top5-acc": 28.49839, "loss": 34.77448, "time": 2.11366}
{"mode": "train", "epoch": 60, "iter": 100, "lr": 0.00992, "memory": 8152, "data_time": 0.25475, "token_loss": 4.47818, "kpt_loss": 41.82493, "top1-acc": 7.77171, "top2-acc": 14.15901, "top5-acc": 28.30687, "loss": 46.30311, "time": 2.33566}
{"mode": "train", "epoch": 60, "iter": 200, "lr": 0.00992, "memory": 8152, "data_time": 0.00051, "token_loss": 4.48866, "kpt_loss": 22.64799, "top1-acc": 7.75287, "top2-acc": 14.10742, "top5-acc": 28.24161, "loss": 27.13665, "time": 2.11163}
{"mode": "train", "epoch": 60, "iter": 300, "lr": 0.00992, "memory": 8152, "data_time": 0.00048, "token_loss": 4.49225, "kpt_loss": 30.63703, "top1-acc": 7.75896, "top2-acc": 14.08801, "top5-acc": 28.18474, "loss": 35.12928, "time": 2.11226}
{"mode": "val", "epoch": 60, "iter": 176, "lr": 0.00992, "Head AP": 86.17283, "Shou AP": 86.60483, "Elb AP": 75.6512, "Wri AP": 68.90444, "Hip AP": 79.3024, "Knee AP": 78.20928, "Ankl AP": 70.50568, "Total AP": 78.45828}
{"mode": "train", "epoch": 61, "iter": 100, "lr": 0.00992, "memory": 8152, "data_time": 0.24009, "token_loss": 4.47239, "kpt_loss": 31.64742, "top1-acc": 7.7724, "top2-acc": 14.12615, "top5-acc": 28.34858, "loss": 36.11981, "time": 2.27334}
{"mode": "train", "epoch": 61, "iter": 200, "lr": 0.00992, "memory": 8152, "data_time": 0.00048, "token_loss": 4.481, "kpt_loss": 18.37364, "top1-acc": 7.83525, "top2-acc": 14.15476, "top5-acc": 28.32169, "loss": 22.85464, "time": 2.10513}
{"mode": "train", "epoch": 61, "iter": 300, "lr": 0.00992, "memory": 8152, "data_time": 0.00044, "token_loss": 4.4722, "kpt_loss": 41.27928, "top1-acc": 7.84341, "top2-acc": 14.26, "top5-acc": 28.52206, "loss": 45.75148, "time": 2.11171}
{"mode": "train", "epoch": 62, "iter": 100, "lr": 0.00991, "memory": 8152, "data_time": 0.25595, "token_loss": 4.46156, "kpt_loss": 24.62168, "top1-acc": 7.89062, "top2-acc": 14.28573, "top5-acc": 28.57284, "loss": 29.08324, "time": 2.33562}
{"mode": "train", "epoch": 62, "iter": 200, "lr": 0.00991, "memory": 8152, "data_time": 0.00045, "token_loss": 4.46127, "kpt_loss": 32.0329, "top1-acc": 7.92957, "top2-acc": 14.38281, "top5-acc": 28.70232, "loss": 36.49417, "time": 2.10998}
{"mode": "train", "epoch": 62, "iter": 300, "lr": 0.00991, "memory": 8152, "data_time": 0.00052, "token_loss": 4.47846, "kpt_loss": 26.49858, "top1-acc": 7.79596, "top2-acc": 14.14063, "top5-acc": 28.28309, "loss": 30.97704, "time": 2.10874}
{"mode": "train", "epoch": 63, "iter": 100, "lr": 0.00991, "memory": 8152, "data_time": 0.25174, "token_loss": 4.47002, "kpt_loss": 19.9197, "top1-acc": 7.90568, "top2-acc": 14.2994, "top5-acc": 28.55802, "loss": 24.38972, "time": 2.33616}
{"mode": "train", "epoch": 63, "iter": 200, "lr": 0.00991, "memory": 8152, "data_time": 0.00045, "token_loss": 4.46216, "kpt_loss": 28.72894, "top1-acc": 7.91314, "top2-acc": 14.31767, "top5-acc": 28.65085, "loss": 33.1911, "time": 2.11747}
{"mode": "train", "epoch": 63, "iter": 300, "lr": 0.00991, "memory": 8152, "data_time": 0.00047, "token_loss": 4.46648, "kpt_loss": 62.91576, "top1-acc": 7.83295, "top2-acc": 14.24667, "top5-acc": 28.50046, "loss": 67.38224, "time": 2.11704}
{"mode": "train", "epoch": 64, "iter": 100, "lr": 0.00991, "memory": 8152, "data_time": 0.25591, "token_loss": 4.47552, "kpt_loss": 30.34807, "top1-acc": 7.81009, "top2-acc": 14.15935, "top5-acc": 28.36432, "loss": 34.82359, "time": 2.33574}
{"mode": "train", "epoch": 64, "iter": 200, "lr": 0.00991, "memory": 8152, "data_time": 0.00045, "token_loss": 4.46522, "kpt_loss": 31.62623, "top1-acc": 7.87098, "top2-acc": 14.3179, "top5-acc": 28.56985, "loss": 36.09145, "time": 2.11151}
{"mode": "train", "epoch": 64, "iter": 300, "lr": 0.00991, "memory": 8152, "data_time": 0.00045, "token_loss": 4.47284, "kpt_loss": 28.50632, "top1-acc": 7.92165, "top2-acc": 14.35892, "top5-acc": 28.58456, "loss": 32.97916, "time": 2.10853}

Also in stage I, we got quiet well accuracy 99.6, but still .... unstable joint_loss we got.

{"mode": "train", "epoch": 1, "iter": 100, "lr": 0.00199, "memory": 1182, "data_time": 0.94688, "joint_loss": 131.75798, "e_latent_loss": 0.29036, "loss": 132.04833, "time": 2.31795}
{"mode": "train", "epoch": 2, "iter": 100, "lr": 0.00578, "memory": 1182, "data_time": 1.3633, "joint_loss": 92.77402, "e_latent_loss": 0.06755, "loss": 92.84157, "time": 2.27915}
{"mode": "train", "epoch": 3, "iter": 100, "lr": 0.00954, "memory": 1182, "data_time": 1.42425, "joint_loss": 54.97908, "e_latent_loss": 0.27928, "loss": 55.25836, "time": 2.30562}
{"mode": "train", "epoch": 4, "iter": 100, "lr": 0.00991, "memory": 1182, "data_time": 0.9994, "joint_loss": 37.38598, "e_latent_loss": 0.83145, "loss": 38.21743, "time": 2.35423}
{"mode": "train", "epoch": 5, "iter": 100, "lr": 0.00984, "memory": 1182, "data_time": 1.21084, "joint_loss": 29.45429, "e_latent_loss": 0.99935, "loss": 30.45364, "time": 2.38208}
{"mode": "val", "epoch": 5, "iter": 88, "lr": 0.00984, "Head AP": 98.64887, "Shou AP": 99.20207, "Elb AP": 99.37901, "Wri AP": 97.74129, "Hip AP": 99.37578, "Knee AP": 99.17843, "Ankl AP": 97.05392, "Total AP": 98.65384}
{"mode": "train", "epoch": 6, "iter": 100, "lr": 0.00976, "memory": 1182, "data_time": 2.17387, "joint_loss": 31.64595, "e_latent_loss": 0.95502, "loss": 32.60097, "time": 2.36026}
{"mode": "train", "epoch": 7, "iter": 100, "lr": 0.00965, "memory": 1182, "data_time": 1.42141, "joint_loss": 21.76995, "e_latent_loss": 0.89484, "loss": 22.66479, "time": 2.31485}
{"mode": "train", "epoch": 8, "iter": 100, "lr": 0.00952, "memory": 1182, "data_time": 1.16861, "joint_loss": 23.02267, "e_latent_loss": 0.83432, "loss": 23.85699, "time": 2.37846}
{"mode": "train", "epoch": 9, "iter": 100, "lr": 0.00938, "memory": 1182, "data_time": 1.44073, "joint_loss": 28.23125, "e_latent_loss": 0.79245, "loss": 29.0237, "time": 2.33312}
{"mode": "train", "epoch": 10, "iter": 100, "lr": 0.00922, "memory": 1182, "data_time": 1.88699, "joint_loss": 18.71602, "e_latent_loss": 0.74751, "loss": 19.46353, "time": 2.31172}
{"mode": "val", "epoch": 10, "iter": 88, "lr": 0.00922, "Head AP": 99.22045, "Shou AP": 99.51017, "Elb AP": 99.54151, "Wri AP": 99.51012, "Hip AP": 99.5415, "Knee AP": 99.48848, "Ankl AP": 99.30971, "Total AP": 99.43095}
{"mode": "train", "epoch": 11, "iter": 100, "lr": 0.00905, "memory": 1182, "data_time": 2.24853, "joint_loss": 23.34516, "e_latent_loss": 0.71205, "loss": 24.05721, "time": 2.40835}
{"mode": "train", "epoch": 12, "iter": 100, "lr": 0.00885, "memory": 1182, "data_time": 1.69354, "joint_loss": 28.24428, "e_latent_loss": 0.68738, "loss": 28.93166, "time": 2.29927}
{"mode": "train", "epoch": 13, "iter": 100, "lr": 0.00864, "memory": 1182, "data_time": 1.96753, "joint_loss": 15.29768, "e_latent_loss": 0.63425, "loss": 15.93193, "time": 2.25577}
{"mode": "train", "epoch": 14, "iter": 100, "lr": 0.00842, "memory": 1182, "data_time": 1.39245, "joint_loss": 22.54629, "e_latent_loss": 0.60328, "loss": 23.14957, "time": 2.32694}
{"mode": "train", "epoch": 15, "iter": 100, "lr": 0.00819, "memory": 1182, "data_time": 1.55896, "joint_loss": 22.6478, "e_latent_loss": 0.5757, "loss": 23.22349, "time": 2.40171}
{"mode": "val", "epoch": 15, "iter": 88, "lr": 0.00819, "Head AP": 99.41775, "Shou AP": 99.5838, "Elb AP": 99.65959, "Wri AP": 99.52926, "Hip AP": 99.66479, "Knee AP": 99.65508, "Ankl AP": 99.37064, "Total AP": 99.54531}
{"mode": "train", "epoch": 16, "iter": 100, "lr": 0.00794, "memory": 1182, "data_time": 2.23116, "joint_loss": 24.33294, "e_latent_loss": 0.54557, "loss": 24.87851, "time": 2.42163}
{"mode": "train", "epoch": 17, "iter": 100, "lr": 0.00768, "memory": 1182, "data_time": 1.39659, "joint_loss": 18.9941, "e_latent_loss": 0.5146, "loss": 19.5087, "time": 2.31948}
{"mode": "train", "epoch": 18, "iter": 100, "lr": 0.00741, "memory": 1182, "data_time": 2.04778, "joint_loss": 33.6255, "e_latent_loss": 0.4894, "loss": 34.11491, "time": 2.34302}
{"mode": "train", "epoch": 19, "iter": 100, "lr": 0.00713, "memory": 1182, "data_time": 1.60108, "joint_loss": 24.88208, "e_latent_loss": 0.45916, "loss": 25.34124, "time": 2.36392}
{"mode": "train", "epoch": 20, "iter": 100, "lr": 0.00684, "memory": 1182, "data_time": 2.20995, "joint_loss": 33.63736, "e_latent_loss": 0.38643, "loss": 34.02379, "time": 2.38279}
{"mode": "val", "epoch": 20, "iter": 88, "lr": 0.00684, "Head AP": 99.34623, "Shou AP": 99.57694, "Elb AP": 99.69345, "Wri AP": 99.61511, "Hip AP": 99.64083, "Knee AP": 99.73131, "Ankl AP": 99.55741, "Total AP": 99.57792}
{"mode": "train", "epoch": 21, "iter": 100, "lr": 0.00655, "memory": 1182, "data_time": 2.27414, "joint_loss": 20.44649, "e_latent_loss": 0.39592, "loss": 20.84241, "time": 2.42805}
{"mode": "train", "epoch": 22, "iter": 100, "lr": 0.00624, "memory": 1182, "data_time": 1.30473, "joint_loss": 20.25056, "e_latent_loss": 0.3664, "loss": 20.61696, "time": 2.45284}
{"mode": "train", "epoch": 23, "iter": 100, "lr": 0.00594, "memory": 1182, "data_time": 1.34377, "joint_loss": 18.64966, "e_latent_loss": 0.32335, "loss": 18.97301, "time": 2.29569}
{"mode": "train", "epoch": 24, "iter": 100, "lr": 0.00563, "memory": 1182, "data_time": 2.19348, "joint_loss": 24.35393, "e_latent_loss": 0.31902, "loss": 24.67295, "time": 2.33287}
{"mode": "train", "epoch": 25, "iter": 100, "lr": 0.00531, "memory": 1182, "data_time": 1.56396, "joint_loss": 25.415, "e_latent_loss": 0.31966, "loss": 25.73466, "time": 2.33948}
{"mode": "val", "epoch": 25, "iter": 88, "lr": 0.00531, "Head AP": 99.43651, "Shou AP": 99.61259, "Elb AP": 99.71928, "Wri AP": 99.61717, "Hip AP": 99.72927, "Knee AP": 99.73076, "Ankl AP": 99.55443, "Total AP": 99.61577}
{"mode": "train", "epoch": 26, "iter": 100, "lr": 0.005, "memory": 1182, "data_time": 2.26028, "joint_loss": 28.60506, "e_latent_loss": 0.31394, "loss": 28.919, "time": 2.435}
{"mode": "train", "epoch": 27, "iter": 100, "lr": 0.00469, "memory": 1182, "data_time": 0.83695, "joint_loss": 17.78252, "e_latent_loss": 0.29297, "loss": 18.07549, "time": 2.45181}
{"mode": "train", "epoch": 28, "iter": 100, "lr": 0.00437, "memory": 1182, "data_time": 0.73813, "joint_loss": 32.04844, "e_latent_loss": 0.28856, "loss": 32.337, "time": 2.42213}
{"mode": "train", "epoch": 29, "iter": 100, "lr": 0.00406, "memory": 1182, "data_time": 0.94718, "joint_loss": 21.95948, "e_latent_loss": 0.288, "loss": 22.24749, "time": 2.39676}
{"mode": "train", "epoch": 30, "iter": 100, "lr": 0.00376, "memory": 1182, "data_time": 1.31979, "joint_loss": 25.35574, "e_latent_loss": 0.29214, "loss": 25.64788, "time": 2.41457}
{"mode": "val", "epoch": 30, "iter": 88, "lr": 0.00376, "Head AP": 99.44268, "Shou AP": 99.64053, "Elb AP": 99.75265, "Wri AP": 99.63487, "Hip AP": 99.72676, "Knee AP": 99.75387, "Ankl AP": 99.59701, "Total AP": 99.63596}
{"mode": "train", "epoch": 31, "iter": 100, "lr": 0.00345, "memory": 1182, "data_time": 2.24078, "joint_loss": 28.22294, "e_latent_loss": 0.29135, "loss": 28.51429, "time": 2.41737}
{"mode": "train", "epoch": 32, "iter": 100, "lr": 0.00316, "memory": 1182, "data_time": 2.36239, "joint_loss": 24.06951, "e_latent_loss": 0.29207, "loss": 24.36158, "time": 2.5463}
{"mode": "train", "epoch": 33, "iter": 100, "lr": 0.00287, "memory": 1182, "data_time": 1.14695, "joint_loss": 33.59997, "e_latent_loss": 0.28858, "loss": 33.88855, "time": 2.35385}
{"mode": "train", "epoch": 34, "iter": 100, "lr": 0.00259, "memory": 1182, "data_time": 1.74583, "joint_loss": 27.6941, "e_latent_loss": 0.29634, "loss": 27.99043, "time": 2.36074}
{"mode": "train", "epoch": 35, "iter": 100, "lr": 0.00232, "memory": 1182, "data_time": 1.74191, "joint_loss": 42.03005, "e_latent_loss": 0.30221, "loss": 42.33226, "time": 2.34875}
{"mode": "val", "epoch": 35, "iter": 88, "lr": 0.00232, "Head AP": 99.44404, "Shou AP": 99.60753, "Elb AP": 99.72326, "Wri AP": 99.63178, "Hip AP": 99.72394, "Knee AP": 99.73707, "Ankl AP": 99.58788, "Total AP": 99.62367}
{"mode": "train", "epoch": 36, "iter": 100, "lr": 0.00206, "memory": 1182, "data_time": 2.33882, "joint_loss": 27.86087, "e_latent_loss": 0.30353, "loss": 28.1644, "time": 2.53479}
{"mode": "train", "epoch": 37, "iter": 100, "lr": 0.00181, "memory": 1182, "data_time": 2.01404, "joint_loss": 25.73541, "e_latent_loss": 0.30623, "loss": 26.04164, "time": 2.31369}
{"mode": "train", "epoch": 38, "iter": 100, "lr": 0.00158, "memory": 1182, "data_time": 2.13694, "joint_loss": 32.76606, "e_latent_loss": 0.30764, "loss": 33.0737, "time": 2.42058}
{"mode": "train", "epoch": 39, "iter": 100, "lr": 0.00136, "memory": 1182, "data_time": 1.79045, "joint_loss": 17.80296, "e_latent_loss": 0.30435, "loss": 18.10731, "time": 2.28525}
{"mode": "train", "epoch": 40, "iter": 100, "lr": 0.00115, "memory": 1182, "data_time": 1.86919, "joint_loss": 17.26738, "e_latent_loss": 0.31324, "loss": 17.58062, "time": 2.35789}
{"mode": "val", "epoch": 40, "iter": 88, "lr": 0.00115, "Head AP": 99.45255, "Shou AP": 99.63473, "Elb AP": 99.75129, "Wri AP": 99.64713, "Hip AP": 99.74225, "Knee AP": 99.76405, "Ankl AP": 99.5946, "Total AP": 99.64172}
{"mode": "train", "epoch": 41, "iter": 100, "lr": 0.00096, "memory": 1182, "data_time": 2.26471, "joint_loss": 31.59792, "e_latent_loss": 0.31512, "loss": 31.91304, "time": 2.44659}
{"mode": "train", "epoch": 42, "iter": 100, "lr": 0.00078, "memory": 1182, "data_time": 0.94218, "joint_loss": 31.79277, "e_latent_loss": 0.31505, "loss": 32.10782, "time": 2.4156}
{"mode": "train", "epoch": 43, "iter": 100, "lr": 0.00062, "memory": 1182, "data_time": 2.21917, "joint_loss": 21.70378, "e_latent_loss": 0.31516, "loss": 22.01894, "time": 2.45668}
{"mode": "train", "epoch": 44, "iter": 100, "lr": 0.00048, "memory": 1182, "data_time": 1.2895, "joint_loss": 14.85339, "e_latent_loss": 0.31228, "loss": 15.16567, "time": 2.46367}
{"mode": "train", "epoch": 45, "iter": 100, "lr": 0.00035, "memory": 1182, "data_time": 2.18394, "joint_loss": 14.99142, "e_latent_loss": 0.31958, "loss": 15.311, "time": 2.49193}
{"mode": "val", "epoch": 45, "iter": 88, "lr": 0.00035, "Head AP": 99.45066, "Shou AP": 99.62896, "Elb AP": 99.72564, "Wri AP": 99.63258, "Hip AP": 99.73462, "Knee AP": 99.78388, "Ankl AP": 99.60328, "Total AP": 99.63799}
{"mode": "train", "epoch": 46, "iter": 100, "lr": 0.00024, "memory": 1182, "data_time": 2.32173, "joint_loss": 23.4917, "e_latent_loss": 0.31459, "loss": 23.80628, "time": 2.51406}
{"mode": "train", "epoch": 47, "iter": 100, "lr": 0.00016, "memory": 1182, "data_time": 2.41419, "joint_loss": 30.79709, "e_latent_loss": 0.32073, "loss": 31.11782, "time": 2.60599}
{"mode": "train", "epoch": 48, "iter": 100, "lr": 9e-05, "memory": 1182, "data_time": 2.19983, "joint_loss": 23.37043, "e_latent_loss": 0.31429, "loss": 23.68472, "time": 2.35457}
{"mode": "train", "epoch": 49, "iter": 100, "lr": 4e-05, "memory": 1182, "data_time": 2.18958, "joint_loss": 16.5158, "e_latent_loss": 0.3193, "loss": 16.8351, "time": 2.38256}
{"mode": "train", "epoch": 50, "iter": 100, "lr": 1e-05, "memory": 1182, "data_time": 2.15379, "joint_loss": 20.5428, "e_latent_loss": 0.30825, "loss": 20.85105, "time": 2.35541}
{"mode": "val", "epoch": 50, "iter": 88, "lr": 1e-05, "Head AP": 99.45537, "Shou AP": 99.63308, "Elb AP": 99.74333, "Wri AP": 99.6398, "Hip AP": 99.75063, "Knee AP": 99.73592, "Ankl AP": 99.58887, "Total AP": 99.63662}

Has anyone run into the same problem?
Or how could I fix it ?