YOLOv3训练自己数据集中所用到的代码及命令总结
程序员文章站
2022-04-29 18:57:30
...
一. 将数据集整理为VOC2007标准格式所需要的一些代码：
1. 将所有图片重命名为000000.jpg的格式：
import os
path = "/home/Leequens/VOC/VOC2007/JPEG/test/YDXJ0013"
path1 = "/home/Leequens/VOC/VOC2007/JPEG/test/test6" # 生成到新路径中
filelist = os.listdir(path) #该文件夹下所有的文件（包括文件夹）
for file in filelist: #遍历所有文件
Olddir=os.path.join(path,file) #原来的文件路径
if os.path.isdir(Olddir): #如果是文件夹则跳过
continue
filename=os.path.splitext(file)[0] #文件名
filetype=os.path.splitext(file)[1] #文件扩展名
Newdir=os.path.join(path1,str(int(filename)+7240).zfill(6)+filetype) #用字符串函数zfill 以0补全所需位数
os.rename(Olddir,Newdir)#重命名2. 根据真值表生成每张图片所对应的xml文件：
# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image
# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/G0024173'
# VEDAI 图像的 ground truth 的 txt 文件存放位置
src_xml_dir = os.path.abspath('.')+'/xml/G0024173'
# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
if os.path.isdir(dir_path):
for x in os.listdir(dir_path):
get_img_list(os.path.join(dir_path, x))
elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
img_Lists.append(dir_path)
get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])
# for i in img_Lists:
# print(i)
# 创建xml文件，存入图片信息
for img_item in img_Lists:
im = Image.open(img_item) #打开图片 为了记录图片的长宽数据
img = os.path.split(img_item)[1].split('.')[0]
width, height = im.size
# write in xml file
# os.mknod(src_xml_dir + '/' + img + '.xml')
xml_file = open((src_xml_dir + '/' + str(img) + '.xml'), 'w')
xml_file.write('<annotation>\n')
xml_file.write(' <folder>VOC2007</folder>\n')
xml_file.write(' <filename>' + str(img) + '.jpg' + '</filename>\n')
xml_file.write(' <size>\n')
xml_file.write(' <width>' + str(width) + '</width>\n')
xml_file.write(' <height>' + str(height) + '</height>\n')
xml_file.write(' <depth>3</depth>\n')
xml_file.write(' </size>\n')
xml_file.close()
# 读取全部信息
txt_file = open('G0024173_chage.txt')
for line in txt_file.readlines():
gt = line.splitlines()
# print(gt)
# gt = txt_file.readline().splitlines()
# # gt = open(src_txt_dir + '/gt_' + img + '.txt').read().splitlines()
# write the region of image on xml file
for img_each_label in gt:
spt = img_each_label.split(' ') # 这里如果txt里面是以逗号‘，’隔开的，那么就改为spt = img_each_label.split(',')。
# 判断是否需要写入xml
if spt[6] == '0': #因为上次比赛给的真值表第六列是loss，所以需要判断
# print (gt)
# 打开相应xml文件，下面具体写入哪一列根据真值表来修改
# print(spt[5].zfill(6))
xml_file = open((src_xml_dir + '/' + spt[5].zfill(6) + '.xml'), 'a')
xml_file.write(' <object>\n')
xml_file.write(' <name>' + str(spt[9]) + '</name>\n')
xml_file.write(' <pose>Unspecified</pose>\n')
xml_file.write(' <truncated>0</truncated>\n')
xml_file.write(' <difficult>0</difficult>\n')
xml_file.write(' <bndbox>\n')
xml_file.write(' <xmin>' + str(spt[1]) + '</xmin>\n')
xml_file.write(' <ymin>' + str(spt[2]) + '</ymin>\n')
xml_file.write(' <xmax>' + str(spt[3]) + '</xmax>\n')
xml_file.write(' <ymax>' + str(spt[4]) + '</ymax>\n')
xml_file.write(' </bndbox>\n')
xml_file.write(' </object>\n')
xml_file.close()
# 补上结尾，数值自己修改
for i in range(1800,3600):
xml_file = open((src_xml_dir + '/' + str(i).zfill(6) + '.xml'), 'a')
xml_file.write('</annotation>')
xml_file.close()二. 一些关于YOLOv3的代码修改：
1. 开始训练命令
先是合并txt文档
cat 2007_train.txt 2007_val.txt > train.txt训练之前要更改voc.data、yolov3-voc.cfg、voc.name、coco.name文件
./darknet detector train cfg/voc.data cfg/yolov3-voc.cfg darknet53.conv.742. 使YOLOv3批量测试图片并保存到文件夹中。
(1) 可以用下面代码替换detector.c文件（example文件夹下）的void test_detector函数。记得有三处要修改路径。
void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filename, float thresh, float hier_thresh, char *outfile, int fullscreen)
{
list *options = read_data_cfg(datacfg);
char *name_list = option_find_str(options, "names", "data/names.list");
char **names = get_labels(name_list);
image **alphabet = load_alphabet();
network *net = load_network(cfgfile, weightfile, 0);
set_batch_network(net, 1);
srand(2222222);
double time;
char buff[256];
char *input = buff;
float nms=.45;
int i=0;
while(1){
if(filename){
strncpy(input, filename, 256);
image im = load_image_color(input,0,0);
image sized = letterbox_image(im, net->w, net->h);
//image sized = resize_image(im, net->w, net->h);
//image sized2 = resize_max(im, net->w);
//image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
//resize_network(net, sized.w, sized.h);
layer l = net->layers[net->n-1];
float *X = sized.data;
time=what_time_is_it_now();
network_predict(net, X);
printf("%s: Predicted in %f seconds.\n", input, what_time_is_it_now()-time);
int nboxes = 0;
detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
//printf("%d\n", nboxes);
//if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
free_detections(dets, nboxes);
if(outfile)
{
save_image(im, outfile);
}
else{
save_image(im, "predictions");
#ifdef OPENCV
cvNamedWindow("predictions", CV_WINDOW_NORMAL);
if(fullscreen){
cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
}
show_image(im, "predictions");
cvWaitKey(0);
cvDestroyAllWindows();
#endif
}
free_image(im);
free_image(sized);
if (filename) break;
}
else {
printf("Enter Image Path: ");
fflush(stdout);
input = fgets(input, 256, stdin);
if(!input) return;
strtok(input, "\n");
list *plist = get_paths(input);
char **paths = (char **)list_to_array(plist);
printf("Start Testing!\n");
int m = plist->size;
if(access("/home/Leequens/darknet/data/out",0)==-1)//"/home/FENGsl/darknet/data"修改成自己的路径
{
if (mkdir("/home/Leequens/darknet/data/out",0777))//"/home/FENGsl/darknet/data"修改成自己的路径
{
printf("creat file bag failed!!!");
}
}
for(i = 0; i < m; ++i){
char *path = paths[i];
image im = load_image_color(path,0,0);
image sized = letterbox_image(im, net->w, net->h);
//image sized = resize_image(im, net->w, net->h);
//image sized2 = resize_max(im, net->w);
//image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
//resize_network(net, sized.w, sized.h);
layer l = net->layers[net->n-1];
float *X = sized.data;
time=what_time_is_it_now();
network_predict(net, X);
printf("Try Very Hard:");
printf("%s: Predicted in %f seconds.\n", path, what_time_is_it_now()-time);
int nboxes = 0;
detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
//printf("%d\n", nboxes);
//if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
free_detections(dets, nboxes);
if(outfile){
save_image(im, outfile);
}
else{
char b[2048];
sprintf(b,"/home/Leequens/darknet/data/out/%s",GetFilename(path));//"/home/FENGsl/darknet/data"修改成自己的路径
save_image(im, b);
printf("save %s successfully!\n",GetFilename(path));
#ifdef OPENCV
cvNamedWindow("predictions", CV_WINDOW_NORMAL);
if(fullscreen){
cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
}
show_image(im, "predictions");
cvWaitKey(0);
cvDestroyAllWindows();
#endif
}
free_image(im);
free_image(sized);
if (filename) break;
}
}
}
}(2) 然后在前面添加*GetFilename(char *p)函数
#include"darknet.h"
#include<sys/stat.h>
#include<stdio.h>
#include<time.h>
#include<unistd.h>
#include<sys/types.h>
static int coco_ids[] = {1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90};
char *GetFilename(char *p)
{
static char name[20]={""};
char *q = strrchr(p,'/') + 1;
strncpy(name,q,6);
return name;
}(3) 重新在darknet下make
(4) 执行下面命令开始测试 (把权重名字修改)
./darknet detector test cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -thresh 0.1
YOLOv3默认阈值为0.25，-thresh 0是设置阈值，设为0的话可以显示出所有检测结果。
运行完直接会让你输入 Enter Image Path:
这里直接输入voc生成好的test文档的绝对路径就可以，里面包含了所有测试集的绝对路径。
3.使YOLOv3输出测试结果保存在txt文档中
./darknet detector valid cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -out detect_result.txt -thresh 0.1
4. 此次目标大赛txt文档格式为 图片id 类id 置信度 4个坐标，需要修改刚刚生成的txt
cat 123.txt | awk '{$7=$6;$6=$5;$5=$4;$4=$3;$3=$2;$2=1; print $0;}' > 123_chage.txt5. 将生成的几个txt合并到一起
cat 1.txt 2.txt 3.txt > result.txt6. 对合并到一起的txt按第一列（图片ID）的值进行排序，目的是使同一图的信息罗列在一起
sort -t $'\t' -k1 -n result.txt > 1result.txt # k1指的是第一列
7. 可以通过修改detector.c文件使得完成3-6步骤
先修改validate_detector中的第二个if语句，直接复制替换
if(0==strcmp(type, "coco")){
if(!outfile) outfile = "coco_results";
snprintf(buff, 1024, "%s/%s.json", prefix, outfile);
fp = fopen(buff, "w");
fprintf(fp, "[\n");
coco = 1;
} else if(0==strcmp(type, "imagenet")){
if(!outfile) outfile = "imagenet-detection";
snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
fp = fopen(buff, "w");
imagenet = 1;
classes = 200;
} else {
if(!outfile) outfile = "comp4_det_test_";
snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
fp = fopen(buff, "w");
}
再修改validate_detector函数中的语句 print_detector_detections(fps, id, dets, nboxes, classes, w, h);
将fps修改为fp：print_detector_detections(fp, id, dets, nboxes, classes, w, h);
再用下面代码替换 print_detector_detections 函数
void print_detector_detections(FILE *fp, char *id, detection *dets, int total, int classes, int w, int h)
{
int i, j;
int nums[4]={1,2,3,4}; #自己class有几类就相应的改掉，这里的1234对应的顺序和voc.name中一样
for(i = 0; i < total; ++i){
float xmin = dets[i].bbox.x - dets[i].bbox.w/2. + 1;
float xmax = dets[i].bbox.x + dets[i].bbox.w/2. + 1;
float ymin = dets[i].bbox.y - dets[i].bbox.h/2. + 1;
float ymax = dets[i].bbox.y + dets[i].bbox.h/2. + 1;
if (xmin < 1) xmin = 1;
if (ymin < 1) ymin = 1;
if (xmax > w) xmax = w;
if (ymax > h) ymax = h;
for(j = 0; j < classes; ++j){
if (dets[i].prob[j]) fprintf(fp, "%s %d %f %f %f %f %f\n", id, nums[j], dets[i].prob[j],
xmin, ymin, xmax, ymax);
}
}
}然后重新make，再执行生成txt文档的语句，就可以把所有class的检测结果输出在一个txt文档中。
PS：make之前记得删除原有的darknet可执行文件
三、处理离线比赛给的xml节点的代码
1. 将xml中的三类object分别替换为1 2 3
import os
import sys
try:
import xml.etree.cElementTree as ET
except ImportError:
import xml.etree.ElementTree as ET
path = "/media/leequens/File/YOLO/test/xml/123" #文件夹路径
filelist = os.listdir(path) #读取该文件夹下的所有文件
for file in filelist: #循环每个文件
filename = os.path.splitext(file)[0] # 文件名
filetype = os.path.splitext(file)[1] # 文件扩展名
tree = ET.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
for elem in tree.iter(tag='name'): # 遍历树中的name节点
if elem.text == '"scallop"':
elem.text = '1'
if elem.text == '"seaurchin"':
elem.text = '2'
if elem.text == '"seacucumber"':
elem.text = '3'
tree.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml') #修改完毕后写入另一个文件夹中2. 将比赛官方给的xml中的frame改为自己重命名后的图片名
from xml.etree import ElementTree
import os
path = "/media/leequens/File/YOLO/test/xml/123"
filelist = os.listdir(path)
for file in filelist:
filename = os.path.splitext(file)[0] # 文件名
filetype = os.path.splitext(file)[1] # 文件扩展名
xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
node= xmldoc.find('./frame') # 遍历树中的movie节点
node.text=str(int(filename)).zfill(6) # 如果需要图片后缀 +'.jpg'
xmldoc.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')
3. 修改xml中节点frame为filename
from xml.etree import ElementTree
import os, sys
import glob
from PIL import Image
path = "/media/leequens/File/YOLO/test/xml/11"
filelist = os.listdir(path)
for file in filelist:
filename = os.path.splitext(file)[0] # 文件名
filetype = os.path.splitext(file)[1] # 文件扩展名
xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/rebox/'+str(int(filename)).zfill(6)+'.xml')
root = xmldoc.getroot()
for child in root:
if child.tag == 'frame':
temp_node = 'filename' # 不懂为啥需要一个临时的变量，直接赋值就不对
child.tag = temp_node
break
xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')
4. 给xml增加图片长宽等信息
from xml.dom import minidom, Node
from xml.etree import ElementTree
import os, sys
import glob
from PIL import Image
path = "/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080"
filelist = os.listdir(path)
for file in filelist:
filename = os.path.splitext(file)[0] # 文件名
filetype = os.path.splitext(file)[1] # 文件扩展名
xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080/'+str(int(filename)).zfill(6)+'.xml')
root = xmldoc.getroot()
folder=ElementTree.Element('folder')
root.insert(0,folder)
folder.text='VOC2007'
size = ElementTree.Element("size")
root.insert(2,size)
width =ElementTree.Element('width')
width.text = '1920'
size.insert(0,width)
height =ElementTree.Element('height')
height.text = '1080'
size.insert(1,height)
depth =ElementTree.Element('depth ')
depth.text = '3'
size.insert(2,depth )
xmldoc.write('/media/leequens/File/YOLO/test/xml/perfectxml/'+str(int(filename)).zfill(6)+'.xml')
数值自己要改，可以用下面代码查看所有图片长宽
# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image
# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/2018image'
# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
if os.path.isdir(dir_path):
for x in os.listdir(dir_path):
get_img_list(os.path.join(dir_path, x))
elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
img_Lists.append(dir_path)
get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])
for img_item in img_Lists:
filename = os.path.splitext(img_item)[0]
im = Image.open(img_item) #打开图片 为了记录图片的长宽数据
img = os.path.split(img_item)[1].split('.')[0]
width, height = im.size
print(filename,width,height)5. 给object加入difficult等节点
from xml.dom import minidom, Node
from xml.etree import ElementTree
import os, sys
import glob
from PIL import Image
path = "/media/leequens/File/YOLO/test/xml/111"
filelist = os.listdir(path)
for file in filelist:
filename = os.path.splitext(file)[0] # 文件名
filetype = os.path.splitext(file)[1] # 文件扩展名
xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')
root = xmldoc.getroot()
pose=ElementTree.Element('pose')
pose.text='Unspecified'
pose.tail='\n\t'
truncated = ElementTree.Element('truncated')
truncated.text='0'
truncated.tail ='\n\t'
difficult = ElementTree.Element('difficult')
difficult.text = '0'
difficult.tail ='\n\t'
for node in xmldoc.iter(tag='object'):
#node = root.find('./object')
#objec=ElementTree.Element("object")
node.insert(1,pose)
node.insert(2,truncated)
node.insert(3,difficult)
xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')
6. 因为测试集是随机分配的，计算mAP时需要提取出测试集所对应的xml文件，测试集的名字都被存储到了test.txt文档里，所以需要编写一个程序把这些xml全部提取出来
import shutil
txt_file= open("111.txt","r",encoding="utf-8",errors="ignore")
while True:
line = txt_file.readline() #表示一次读取一行
line=line.strip('\n') # 这里要删除换行符，不然找不到对应文件，头疼
if not line: #读到数据最后跳出，结束循环。数据的最后也就是读不到数据了，mystr为空的时候
break
oldname=u"/media/leequens/File/YOLO/test/xml/111/"+line+".xml" #旧目录
newname=u"/media/leequens/File/YOLO/test/xml/"+line+".xml" # 新目录
shutil.copyfile(oldname, newname)
./darknet detector demo cfg/voc.data yolov3-voc.cfg yolov3-voc_20000.weights "http://192.168.254.1:8090/?action=stream"
上一篇： Java基础（1）
下一篇： day11_前10天所用到的函数