mirror of
https://github.com/aigc3d/LAM_Audio2Expression.git
synced 2026-02-04 09:29:24 +08:00
578 lines
26 KiB
Python
578 lines
26 KiB
Python
"""
|
|
The code is base on https://github.com/Pointcept/Pointcept
|
|
"""
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.distributed as dist
|
|
from uuid import uuid4
|
|
|
|
import utils.comm as comm
|
|
from utils.misc import intersection_and_union_gpu
|
|
|
|
from .default import HookBase
|
|
from .builder import HOOKS
|
|
|
|
|
|
@HOOKS.register_module()
|
|
class ClsEvaluator(HookBase):
|
|
def after_epoch(self):
|
|
if self.trainer.cfg.evaluate:
|
|
self.eval()
|
|
|
|
def eval(self):
|
|
self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>")
|
|
self.trainer.model.eval()
|
|
for i, input_dict in enumerate(self.trainer.val_loader):
|
|
for key in input_dict.keys():
|
|
if isinstance(input_dict[key], torch.Tensor):
|
|
input_dict[key] = input_dict[key].cuda(non_blocking=True)
|
|
with torch.no_grad():
|
|
output_dict = self.trainer.model(input_dict)
|
|
output = output_dict["cls_logits"]
|
|
loss = output_dict["loss"]
|
|
pred = output.max(1)[1]
|
|
label = input_dict["category"]
|
|
intersection, union, target = intersection_and_union_gpu(
|
|
pred,
|
|
label,
|
|
self.trainer.cfg.data.num_classes,
|
|
self.trainer.cfg.data.ignore_index,
|
|
)
|
|
if comm.get_world_size() > 1:
|
|
dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce(
|
|
target
|
|
)
|
|
intersection, union, target = (
|
|
intersection.cpu().numpy(),
|
|
union.cpu().numpy(),
|
|
target.cpu().numpy(),
|
|
)
|
|
# Here there is no need to sync since sync happened in dist.all_reduce
|
|
self.trainer.storage.put_scalar("val_intersection", intersection)
|
|
self.trainer.storage.put_scalar("val_union", union)
|
|
self.trainer.storage.put_scalar("val_target", target)
|
|
self.trainer.storage.put_scalar("val_loss", loss.item())
|
|
self.trainer.logger.info(
|
|
"Test: [{iter}/{max_iter}] "
|
|
"Loss {loss:.4f} ".format(
|
|
iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item()
|
|
)
|
|
)
|
|
loss_avg = self.trainer.storage.history("val_loss").avg
|
|
intersection = self.trainer.storage.history("val_intersection").total
|
|
union = self.trainer.storage.history("val_union").total
|
|
target = self.trainer.storage.history("val_target").total
|
|
iou_class = intersection / (union + 1e-10)
|
|
acc_class = intersection / (target + 1e-10)
|
|
m_iou = np.mean(iou_class)
|
|
m_acc = np.mean(acc_class)
|
|
all_acc = sum(intersection) / (sum(target) + 1e-10)
|
|
self.trainer.logger.info(
|
|
"Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format(
|
|
m_iou, m_acc, all_acc
|
|
)
|
|
)
|
|
for i in range(self.trainer.cfg.data.num_classes):
|
|
self.trainer.logger.info(
|
|
"Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format(
|
|
idx=i,
|
|
name=self.trainer.cfg.data.names[i],
|
|
iou=iou_class[i],
|
|
accuracy=acc_class[i],
|
|
)
|
|
)
|
|
current_epoch = self.trainer.epoch + 1
|
|
if self.trainer.writer is not None:
|
|
self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch)
|
|
self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch)
|
|
self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch)
|
|
self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch)
|
|
self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<")
|
|
self.trainer.comm_info["current_metric_value"] = all_acc # save for saver
|
|
self.trainer.comm_info["current_metric_name"] = "allAcc" # save for saver
|
|
|
|
def after_train(self):
|
|
self.trainer.logger.info(
|
|
"Best {}: {:.4f}".format("allAcc", self.trainer.best_metric_value)
|
|
)
|
|
|
|
|
|
@HOOKS.register_module()
|
|
class SemSegEvaluator(HookBase):
|
|
def after_epoch(self):
|
|
if self.trainer.cfg.evaluate:
|
|
self.eval()
|
|
|
|
def eval(self):
|
|
self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>")
|
|
self.trainer.model.eval()
|
|
for i, input_dict in enumerate(self.trainer.val_loader):
|
|
for key in input_dict.keys():
|
|
if isinstance(input_dict[key], torch.Tensor):
|
|
input_dict[key] = input_dict[key].cuda(non_blocking=True)
|
|
with torch.no_grad():
|
|
output_dict = self.trainer.model(input_dict)
|
|
output = output_dict["seg_logits"]
|
|
loss = output_dict["loss"]
|
|
pred = output.max(1)[1]
|
|
segment = input_dict["segment"]
|
|
if "origin_coord" in input_dict.keys():
|
|
idx, _ = pointops.knn_query(
|
|
1,
|
|
input_dict["coord"].float(),
|
|
input_dict["offset"].int(),
|
|
input_dict["origin_coord"].float(),
|
|
input_dict["origin_offset"].int(),
|
|
)
|
|
pred = pred[idx.flatten().long()]
|
|
segment = input_dict["origin_segment"]
|
|
intersection, union, target = intersection_and_union_gpu(
|
|
pred,
|
|
segment,
|
|
self.trainer.cfg.data.num_classes,
|
|
self.trainer.cfg.data.ignore_index,
|
|
)
|
|
if comm.get_world_size() > 1:
|
|
dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce(
|
|
target
|
|
)
|
|
intersection, union, target = (
|
|
intersection.cpu().numpy(),
|
|
union.cpu().numpy(),
|
|
target.cpu().numpy(),
|
|
)
|
|
# Here there is no need to sync since sync happened in dist.all_reduce
|
|
self.trainer.storage.put_scalar("val_intersection", intersection)
|
|
self.trainer.storage.put_scalar("val_union", union)
|
|
self.trainer.storage.put_scalar("val_target", target)
|
|
self.trainer.storage.put_scalar("val_loss", loss.item())
|
|
info = "Test: [{iter}/{max_iter}] ".format(
|
|
iter=i + 1, max_iter=len(self.trainer.val_loader)
|
|
)
|
|
if "origin_coord" in input_dict.keys():
|
|
info = "Interp. " + info
|
|
self.trainer.logger.info(
|
|
info
|
|
+ "Loss {loss:.4f} ".format(
|
|
iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item()
|
|
)
|
|
)
|
|
loss_avg = self.trainer.storage.history("val_loss").avg
|
|
intersection = self.trainer.storage.history("val_intersection").total
|
|
union = self.trainer.storage.history("val_union").total
|
|
target = self.trainer.storage.history("val_target").total
|
|
iou_class = intersection / (union + 1e-10)
|
|
acc_class = intersection / (target + 1e-10)
|
|
m_iou = np.mean(iou_class)
|
|
m_acc = np.mean(acc_class)
|
|
all_acc = sum(intersection) / (sum(target) + 1e-10)
|
|
self.trainer.logger.info(
|
|
"Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format(
|
|
m_iou, m_acc, all_acc
|
|
)
|
|
)
|
|
for i in range(self.trainer.cfg.data.num_classes):
|
|
self.trainer.logger.info(
|
|
"Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format(
|
|
idx=i,
|
|
name=self.trainer.cfg.data.names[i],
|
|
iou=iou_class[i],
|
|
accuracy=acc_class[i],
|
|
)
|
|
)
|
|
current_epoch = self.trainer.epoch + 1
|
|
if self.trainer.writer is not None:
|
|
self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch)
|
|
self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch)
|
|
self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch)
|
|
self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch)
|
|
self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<")
|
|
self.trainer.comm_info["current_metric_value"] = m_iou # save for saver
|
|
self.trainer.comm_info["current_metric_name"] = "mIoU" # save for saver
|
|
|
|
def after_train(self):
|
|
self.trainer.logger.info(
|
|
"Best {}: {:.4f}".format("mIoU", self.trainer.best_metric_value)
|
|
)
|
|
|
|
|
|
@HOOKS.register_module()
|
|
class InsSegEvaluator(HookBase):
|
|
def __init__(self, segment_ignore_index=(-1,), instance_ignore_index=-1):
|
|
self.segment_ignore_index = segment_ignore_index
|
|
self.instance_ignore_index = instance_ignore_index
|
|
|
|
self.valid_class_names = None # update in before train
|
|
self.overlaps = np.append(np.arange(0.5, 0.95, 0.05), 0.25)
|
|
self.min_region_sizes = 100
|
|
self.distance_threshes = float("inf")
|
|
self.distance_confs = -float("inf")
|
|
|
|
def before_train(self):
|
|
self.valid_class_names = [
|
|
self.trainer.cfg.data.names[i]
|
|
for i in range(self.trainer.cfg.data.num_classes)
|
|
if i not in self.segment_ignore_index
|
|
]
|
|
|
|
def after_epoch(self):
|
|
if self.trainer.cfg.evaluate:
|
|
self.eval()
|
|
|
|
def associate_instances(self, pred, segment, instance):
|
|
segment = segment.cpu().numpy()
|
|
instance = instance.cpu().numpy()
|
|
void_mask = np.in1d(segment, self.segment_ignore_index)
|
|
|
|
assert (
|
|
pred["pred_classes"].shape[0]
|
|
== pred["pred_scores"].shape[0]
|
|
== pred["pred_masks"].shape[0]
|
|
)
|
|
assert pred["pred_masks"].shape[1] == segment.shape[0] == instance.shape[0]
|
|
# get gt instances
|
|
gt_instances = dict()
|
|
for i in range(self.trainer.cfg.data.num_classes):
|
|
if i not in self.segment_ignore_index:
|
|
gt_instances[self.trainer.cfg.data.names[i]] = []
|
|
instance_ids, idx, counts = np.unique(
|
|
instance, return_index=True, return_counts=True
|
|
)
|
|
segment_ids = segment[idx]
|
|
for i in range(len(instance_ids)):
|
|
if instance_ids[i] == self.instance_ignore_index:
|
|
continue
|
|
if segment_ids[i] in self.segment_ignore_index:
|
|
continue
|
|
gt_inst = dict()
|
|
gt_inst["instance_id"] = instance_ids[i]
|
|
gt_inst["segment_id"] = segment_ids[i]
|
|
gt_inst["dist_conf"] = 0.0
|
|
gt_inst["med_dist"] = -1.0
|
|
gt_inst["vert_count"] = counts[i]
|
|
gt_inst["matched_pred"] = []
|
|
gt_instances[self.trainer.cfg.data.names[segment_ids[i]]].append(gt_inst)
|
|
|
|
# get pred instances and associate with gt
|
|
pred_instances = dict()
|
|
for i in range(self.trainer.cfg.data.num_classes):
|
|
if i not in self.segment_ignore_index:
|
|
pred_instances[self.trainer.cfg.data.names[i]] = []
|
|
instance_id = 0
|
|
for i in range(len(pred["pred_classes"])):
|
|
if pred["pred_classes"][i] in self.segment_ignore_index:
|
|
continue
|
|
pred_inst = dict()
|
|
pred_inst["uuid"] = uuid4()
|
|
pred_inst["instance_id"] = instance_id
|
|
pred_inst["segment_id"] = pred["pred_classes"][i]
|
|
pred_inst["confidence"] = pred["pred_scores"][i]
|
|
pred_inst["mask"] = np.not_equal(pred["pred_masks"][i], 0)
|
|
pred_inst["vert_count"] = np.count_nonzero(pred_inst["mask"])
|
|
pred_inst["void_intersection"] = np.count_nonzero(
|
|
np.logical_and(void_mask, pred_inst["mask"])
|
|
)
|
|
if pred_inst["vert_count"] < self.min_region_sizes:
|
|
continue # skip if empty
|
|
segment_name = self.trainer.cfg.data.names[pred_inst["segment_id"]]
|
|
matched_gt = []
|
|
for gt_idx, gt_inst in enumerate(gt_instances[segment_name]):
|
|
intersection = np.count_nonzero(
|
|
np.logical_and(
|
|
instance == gt_inst["instance_id"], pred_inst["mask"]
|
|
)
|
|
)
|
|
if intersection > 0:
|
|
gt_inst_ = gt_inst.copy()
|
|
pred_inst_ = pred_inst.copy()
|
|
gt_inst_["intersection"] = intersection
|
|
pred_inst_["intersection"] = intersection
|
|
matched_gt.append(gt_inst_)
|
|
gt_inst["matched_pred"].append(pred_inst_)
|
|
pred_inst["matched_gt"] = matched_gt
|
|
pred_instances[segment_name].append(pred_inst)
|
|
instance_id += 1
|
|
return gt_instances, pred_instances
|
|
|
|
def evaluate_matches(self, scenes):
|
|
overlaps = self.overlaps
|
|
min_region_sizes = [self.min_region_sizes]
|
|
dist_threshes = [self.distance_threshes]
|
|
dist_confs = [self.distance_confs]
|
|
|
|
# results: class x overlap
|
|
ap_table = np.zeros(
|
|
(len(dist_threshes), len(self.valid_class_names), len(overlaps)), float
|
|
)
|
|
for di, (min_region_size, distance_thresh, distance_conf) in enumerate(
|
|
zip(min_region_sizes, dist_threshes, dist_confs)
|
|
):
|
|
for oi, overlap_th in enumerate(overlaps):
|
|
pred_visited = {}
|
|
for scene in scenes:
|
|
for _ in scene["pred"]:
|
|
for label_name in self.valid_class_names:
|
|
for p in scene["pred"][label_name]:
|
|
if "uuid" in p:
|
|
pred_visited[p["uuid"]] = False
|
|
for li, label_name in enumerate(self.valid_class_names):
|
|
y_true = np.empty(0)
|
|
y_score = np.empty(0)
|
|
hard_false_negatives = 0
|
|
has_gt = False
|
|
has_pred = False
|
|
for scene in scenes:
|
|
pred_instances = scene["pred"][label_name]
|
|
gt_instances = scene["gt"][label_name]
|
|
# filter groups in ground truth
|
|
gt_instances = [
|
|
gt
|
|
for gt in gt_instances
|
|
if gt["vert_count"] >= min_region_size
|
|
and gt["med_dist"] <= distance_thresh
|
|
and gt["dist_conf"] >= distance_conf
|
|
]
|
|
if gt_instances:
|
|
has_gt = True
|
|
if pred_instances:
|
|
has_pred = True
|
|
|
|
cur_true = np.ones(len(gt_instances))
|
|
cur_score = np.ones(len(gt_instances)) * (-float("inf"))
|
|
cur_match = np.zeros(len(gt_instances), dtype=bool)
|
|
# collect matches
|
|
for gti, gt in enumerate(gt_instances):
|
|
found_match = False
|
|
for pred in gt["matched_pred"]:
|
|
# greedy assignments
|
|
if pred_visited[pred["uuid"]]:
|
|
continue
|
|
overlap = float(pred["intersection"]) / (
|
|
gt["vert_count"]
|
|
+ pred["vert_count"]
|
|
- pred["intersection"]
|
|
)
|
|
if overlap > overlap_th:
|
|
confidence = pred["confidence"]
|
|
# if already have a prediction for this gt,
|
|
# the prediction with the lower score is automatically a false positive
|
|
if cur_match[gti]:
|
|
max_score = max(cur_score[gti], confidence)
|
|
min_score = min(cur_score[gti], confidence)
|
|
cur_score[gti] = max_score
|
|
# append false positive
|
|
cur_true = np.append(cur_true, 0)
|
|
cur_score = np.append(cur_score, min_score)
|
|
cur_match = np.append(cur_match, True)
|
|
# otherwise set score
|
|
else:
|
|
found_match = True
|
|
cur_match[gti] = True
|
|
cur_score[gti] = confidence
|
|
pred_visited[pred["uuid"]] = True
|
|
if not found_match:
|
|
hard_false_negatives += 1
|
|
# remove non-matched ground truth instances
|
|
cur_true = cur_true[cur_match]
|
|
cur_score = cur_score[cur_match]
|
|
|
|
# collect non-matched predictions as false positive
|
|
for pred in pred_instances:
|
|
found_gt = False
|
|
for gt in pred["matched_gt"]:
|
|
overlap = float(gt["intersection"]) / (
|
|
gt["vert_count"]
|
|
+ pred["vert_count"]
|
|
- gt["intersection"]
|
|
)
|
|
if overlap > overlap_th:
|
|
found_gt = True
|
|
break
|
|
if not found_gt:
|
|
num_ignore = pred["void_intersection"]
|
|
for gt in pred["matched_gt"]:
|
|
if gt["segment_id"] in self.segment_ignore_index:
|
|
num_ignore += gt["intersection"]
|
|
# small ground truth instances
|
|
if (
|
|
gt["vert_count"] < min_region_size
|
|
or gt["med_dist"] > distance_thresh
|
|
or gt["dist_conf"] < distance_conf
|
|
):
|
|
num_ignore += gt["intersection"]
|
|
proportion_ignore = (
|
|
float(num_ignore) / pred["vert_count"]
|
|
)
|
|
# if not ignored append false positive
|
|
if proportion_ignore <= overlap_th:
|
|
cur_true = np.append(cur_true, 0)
|
|
confidence = pred["confidence"]
|
|
cur_score = np.append(cur_score, confidence)
|
|
|
|
# append to overall results
|
|
y_true = np.append(y_true, cur_true)
|
|
y_score = np.append(y_score, cur_score)
|
|
|
|
# compute average precision
|
|
if has_gt and has_pred:
|
|
# compute precision recall curve first
|
|
|
|
# sorting and cumsum
|
|
score_arg_sort = np.argsort(y_score)
|
|
y_score_sorted = y_score[score_arg_sort]
|
|
y_true_sorted = y_true[score_arg_sort]
|
|
y_true_sorted_cumsum = np.cumsum(y_true_sorted)
|
|
|
|
# unique thresholds
|
|
(thresholds, unique_indices) = np.unique(
|
|
y_score_sorted, return_index=True
|
|
)
|
|
num_prec_recall = len(unique_indices) + 1
|
|
|
|
# prepare precision recall
|
|
num_examples = len(y_score_sorted)
|
|
# https://github.com/ScanNet/ScanNet/pull/26
|
|
# all predictions are non-matched but also all of them are ignored and not counted as FP
|
|
# y_true_sorted_cumsum is empty
|
|
# num_true_examples = y_true_sorted_cumsum[-1]
|
|
num_true_examples = (
|
|
y_true_sorted_cumsum[-1]
|
|
if len(y_true_sorted_cumsum) > 0
|
|
else 0
|
|
)
|
|
precision = np.zeros(num_prec_recall)
|
|
recall = np.zeros(num_prec_recall)
|
|
|
|
# deal with the first point
|
|
y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0)
|
|
# deal with remaining
|
|
for idx_res, idx_scores in enumerate(unique_indices):
|
|
cumsum = y_true_sorted_cumsum[idx_scores - 1]
|
|
tp = num_true_examples - cumsum
|
|
fp = num_examples - idx_scores - tp
|
|
fn = cumsum + hard_false_negatives
|
|
p = float(tp) / (tp + fp)
|
|
r = float(tp) / (tp + fn)
|
|
precision[idx_res] = p
|
|
recall[idx_res] = r
|
|
|
|
# first point in curve is artificial
|
|
precision[-1] = 1.0
|
|
recall[-1] = 0.0
|
|
|
|
# compute average of precision-recall curve
|
|
recall_for_conv = np.copy(recall)
|
|
recall_for_conv = np.append(recall_for_conv[0], recall_for_conv)
|
|
recall_for_conv = np.append(recall_for_conv, 0.0)
|
|
|
|
stepWidths = np.convolve(
|
|
recall_for_conv, [-0.5, 0, 0.5], "valid"
|
|
)
|
|
# integrate is now simply a dot product
|
|
ap_current = np.dot(precision, stepWidths)
|
|
|
|
elif has_gt:
|
|
ap_current = 0.0
|
|
else:
|
|
ap_current = float("nan")
|
|
ap_table[di, li, oi] = ap_current
|
|
d_inf = 0
|
|
o50 = np.where(np.isclose(self.overlaps, 0.5))
|
|
o25 = np.where(np.isclose(self.overlaps, 0.25))
|
|
oAllBut25 = np.where(np.logical_not(np.isclose(self.overlaps, 0.25)))
|
|
ap_scores = dict()
|
|
ap_scores["all_ap"] = np.nanmean(ap_table[d_inf, :, oAllBut25])
|
|
ap_scores["all_ap_50%"] = np.nanmean(ap_table[d_inf, :, o50])
|
|
ap_scores["all_ap_25%"] = np.nanmean(ap_table[d_inf, :, o25])
|
|
ap_scores["classes"] = {}
|
|
for li, label_name in enumerate(self.valid_class_names):
|
|
ap_scores["classes"][label_name] = {}
|
|
ap_scores["classes"][label_name]["ap"] = np.average(
|
|
ap_table[d_inf, li, oAllBut25]
|
|
)
|
|
ap_scores["classes"][label_name]["ap50%"] = np.average(
|
|
ap_table[d_inf, li, o50]
|
|
)
|
|
ap_scores["classes"][label_name]["ap25%"] = np.average(
|
|
ap_table[d_inf, li, o25]
|
|
)
|
|
return ap_scores
|
|
|
|
def eval(self):
|
|
self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>")
|
|
self.trainer.model.eval()
|
|
scenes = []
|
|
for i, input_dict in enumerate(self.trainer.val_loader):
|
|
assert (
|
|
len(input_dict["offset"]) == 1
|
|
) # currently only support bs 1 for each GPU
|
|
for key in input_dict.keys():
|
|
if isinstance(input_dict[key], torch.Tensor):
|
|
input_dict[key] = input_dict[key].cuda(non_blocking=True)
|
|
with torch.no_grad():
|
|
output_dict = self.trainer.model(input_dict)
|
|
|
|
loss = output_dict["loss"]
|
|
|
|
segment = input_dict["segment"]
|
|
instance = input_dict["instance"]
|
|
# map to origin
|
|
if "origin_coord" in input_dict.keys():
|
|
idx, _ = pointops.knn_query(
|
|
1,
|
|
input_dict["coord"].float(),
|
|
input_dict["offset"].int(),
|
|
input_dict["origin_coord"].float(),
|
|
input_dict["origin_offset"].int(),
|
|
)
|
|
idx = idx.cpu().flatten().long()
|
|
output_dict["pred_masks"] = output_dict["pred_masks"][:, idx]
|
|
segment = input_dict["origin_segment"]
|
|
instance = input_dict["origin_instance"]
|
|
|
|
gt_instances, pred_instance = self.associate_instances(
|
|
output_dict, segment, instance
|
|
)
|
|
scenes.append(dict(gt=gt_instances, pred=pred_instance))
|
|
|
|
self.trainer.storage.put_scalar("val_loss", loss.item())
|
|
self.trainer.logger.info(
|
|
"Test: [{iter}/{max_iter}] "
|
|
"Loss {loss:.4f} ".format(
|
|
iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item()
|
|
)
|
|
)
|
|
|
|
loss_avg = self.trainer.storage.history("val_loss").avg
|
|
comm.synchronize()
|
|
scenes_sync = comm.gather(scenes, dst=0)
|
|
scenes = [scene for scenes_ in scenes_sync for scene in scenes_]
|
|
ap_scores = self.evaluate_matches(scenes)
|
|
all_ap = ap_scores["all_ap"]
|
|
all_ap_50 = ap_scores["all_ap_50%"]
|
|
all_ap_25 = ap_scores["all_ap_25%"]
|
|
self.trainer.logger.info(
|
|
"Val result: mAP/AP50/AP25 {:.4f}/{:.4f}/{:.4f}.".format(
|
|
all_ap, all_ap_50, all_ap_25
|
|
)
|
|
)
|
|
for i, label_name in enumerate(self.valid_class_names):
|
|
ap = ap_scores["classes"][label_name]["ap"]
|
|
ap_50 = ap_scores["classes"][label_name]["ap50%"]
|
|
ap_25 = ap_scores["classes"][label_name]["ap25%"]
|
|
self.trainer.logger.info(
|
|
"Class_{idx}-{name} Result: AP/AP50/AP25 {AP:.4f}/{AP50:.4f}/{AP25:.4f}".format(
|
|
idx=i, name=label_name, AP=ap, AP50=ap_50, AP25=ap_25
|
|
)
|
|
)
|
|
current_epoch = self.trainer.epoch + 1
|
|
if self.trainer.writer is not None:
|
|
self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch)
|
|
self.trainer.writer.add_scalar("val/mAP", all_ap, current_epoch)
|
|
self.trainer.writer.add_scalar("val/AP50", all_ap_50, current_epoch)
|
|
self.trainer.writer.add_scalar("val/AP25", all_ap_25, current_epoch)
|
|
self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<")
|
|
self.trainer.comm_info["current_metric_value"] = all_ap_50 # save for saver
|
|
self.trainer.comm_info["current_metric_name"] = "AP50" # save for saver
|