欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

YOLOv3训练自己数据集中所用到的代码及命令总结

程序员文章站 2022-04-29 18:57:30
...

一. 将数据集整理为VOC2007标准格式所需要的一些代码:

1. 将所有图片重命名为000000.jpg的格式:

import os
path = "/home/Leequens/VOC/VOC2007/JPEG/test/YDXJ0013"
path1 = "/home/Leequens/VOC/VOC2007/JPEG/test/test6" # 生成到新路径中
filelist = os.listdir(path) #该文件夹下所有的文件(包括文件夹)
for file in filelist:   #遍历所有文件
    Olddir=os.path.join(path,file)   #原来的文件路径
    if os.path.isdir(Olddir):   #如果是文件夹则跳过
        continue
    filename=os.path.splitext(file)[0]   #文件名
    filetype=os.path.splitext(file)[1]   #文件扩展名
    Newdir=os.path.join(path1,str(int(filename)+7240).zfill(6)+filetype)  #用字符串函数zfill 以0补全所需位数
    os.rename(Olddir,Newdir)#重命名

2. 根据真值表生成每张图片所对应的xml文件:

# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image

# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/G0024173'
# VEDAI 图像的 ground truth 的 txt 文件存放位置
src_xml_dir = os.path.abspath('.')+'/xml/G0024173'


# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
    if os.path.isdir(dir_path):
        for x in os.listdir(dir_path):
            get_img_list(os.path.join(dir_path, x))
    elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
        img_Lists.append(dir_path)

get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])
# for i in img_Lists:
#     print(i)

# 创建xml文件,存入图片信息
for img_item in img_Lists:
    im = Image.open(img_item)  #打开图片 为了记录图片的长宽数据
    img = os.path.split(img_item)[1].split('.')[0]
    width, height = im.size

    # write in xml file
    # os.mknod(src_xml_dir + '/' + img + '.xml')
    xml_file = open((src_xml_dir + '/' + str(img) + '.xml'), 'w')
    xml_file.write('<annotation>\n')
    xml_file.write('    <folder>VOC2007</folder>\n')
    xml_file.write('    <filename>' + str(img) + '.jpg' + '</filename>\n')
    xml_file.write('    <size>\n')
    xml_file.write('        <width>' + str(width) + '</width>\n')
    xml_file.write('        <height>' + str(height) + '</height>\n')
    xml_file.write('        <depth>3</depth>\n')
    xml_file.write('    </size>\n')
    xml_file.close()

    

# 读取全部信息
txt_file = open('G0024173_chage.txt')

for line in txt_file.readlines():
    gt = line.splitlines()
    # print(gt)
#     gt = txt_file.readline().splitlines()
#     # gt = open(src_txt_dir + '/gt_' + img + '.txt').read().splitlines()

    # write the region of image on xml file
    for img_each_label in gt:
        spt = img_each_label.split(' ')  # 这里如果txt里面是以逗号‘,’隔开的,那么就改为spt = img_each_label.split(',')。

        # 判断是否需要写入xml
        if spt[6] == '0': #因为上次比赛给的真值表第六列是loss,所以需要判断
            # print (gt)

            # 打开相应xml文件,下面具体写入哪一列根据真值表来修改
            # print(spt[5].zfill(6))
            xml_file = open((src_xml_dir + '/' + spt[5].zfill(6) + '.xml'), 'a') 
            xml_file.write('    <object>\n')
            xml_file.write('        <name>' + str(spt[9]) + '</name>\n')
            xml_file.write('        <pose>Unspecified</pose>\n')
            xml_file.write('        <truncated>0</truncated>\n')
            xml_file.write('        <difficult>0</difficult>\n')
            xml_file.write('        <bndbox>\n')
            xml_file.write('            <xmin>' + str(spt[1]) + '</xmin>\n')
            xml_file.write('            <ymin>' + str(spt[2]) + '</ymin>\n')
            xml_file.write('            <xmax>' + str(spt[3]) + '</xmax>\n')
            xml_file.write('            <ymax>' + str(spt[4]) + '</ymax>\n')
            xml_file.write('        </bndbox>\n')
            xml_file.write('    </object>\n')
            xml_file.close()

# 补上结尾,数值自己修改
for i in range(1800,3600): 
    xml_file = open((src_xml_dir + '/' + str(i).zfill(6) + '.xml'), 'a')
    xml_file.write('</annotation>')
    xml_file.close()

二.  一些关于YOLOv3的代码修改:

1. 开始训练命令

  先是合并txt文档

cat 2007_train.txt 2007_val.txt > train.txt

  训练之前要更改voc.data、yolov3-voc.cfg、voc.name、coco.name文件

./darknet detector train cfg/voc.data cfg/yolov3-voc.cfg darknet53.conv.74

2. 使YOLOv3批量测试图片并保存到文件夹中。

(1) 可以用下面代码替换detector.c文件(example文件夹下)的void test_detector函数。记得有三处要修改路径。

void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filename, float thresh, float hier_thresh, char *outfile, int fullscreen)
{
    list *options = read_data_cfg(datacfg);
    char *name_list = option_find_str(options, "names", "data/names.list");
    char **names = get_labels(name_list);
 
    image **alphabet = load_alphabet();
    network *net = load_network(cfgfile, weightfile, 0);
    set_batch_network(net, 1);
    srand(2222222);
    double time;
    char buff[256];
    char *input = buff;
    float nms=.45;
    int i=0;
    while(1){
        if(filename){
            strncpy(input, filename, 256);
            image im = load_image_color(input,0,0);
            image sized = letterbox_image(im, net->w, net->h);
        //image sized = resize_image(im, net->w, net->h);
        //image sized2 = resize_max(im, net->w);
        //image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
        //resize_network(net, sized.w, sized.h);
            layer l = net->layers[net->n-1];
 
 
            float *X = sized.data;
            time=what_time_is_it_now();
            network_predict(net, X);
            printf("%s: Predicted in %f seconds.\n", input, what_time_is_it_now()-time);
            int nboxes = 0;
            detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
            //printf("%d\n", nboxes);
            //if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
            if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
                draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
                free_detections(dets, nboxes);
            if(outfile)
             {
                save_image(im, outfile);
             }
            else{
                save_image(im, "predictions");
#ifdef OPENCV
                cvNamedWindow("predictions", CV_WINDOW_NORMAL); 
                if(fullscreen){
                cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
                }
                show_image(im, "predictions");
                cvWaitKey(0);
                cvDestroyAllWindows();
#endif
            }
            free_image(im);
            free_image(sized);
            if (filename) break;
         } 
        else {
            printf("Enter Image Path: ");
            fflush(stdout);
            input = fgets(input, 256, stdin);
            if(!input) return;
            strtok(input, "\n");
   
            list *plist = get_paths(input);
            char **paths = (char **)list_to_array(plist);
             printf("Start Testing!\n");
            int m = plist->size;
            if(access("/home/Leequens/darknet/data/out",0)==-1)//"/home/FENGsl/darknet/data"修改成自己的路径
            {
              if (mkdir("/home/Leequens/darknet/data/out",0777))//"/home/FENGsl/darknet/data"修改成自己的路径
               {
                 printf("creat file bag failed!!!");
               }
            }
            for(i = 0; i < m; ++i){
             char *path = paths[i];
             image im = load_image_color(path,0,0);
             image sized = letterbox_image(im, net->w, net->h);
        //image sized = resize_image(im, net->w, net->h);
        //image sized2 = resize_max(im, net->w);
        //image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
        //resize_network(net, sized.w, sized.h);
        layer l = net->layers[net->n-1];
 
 
        float *X = sized.data;
        time=what_time_is_it_now();
        network_predict(net, X);
        printf("Try Very Hard:");
        printf("%s: Predicted in %f seconds.\n", path, what_time_is_it_now()-time);
        int nboxes = 0;
        detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
        //printf("%d\n", nboxes);
        //if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
        if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
        draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
        free_detections(dets, nboxes);
        if(outfile){
            save_image(im, outfile);
        }
        else{
             
             char b[2048];
            sprintf(b,"/home/Leequens/darknet/data/out/%s",GetFilename(path));//"/home/FENGsl/darknet/data"修改成自己的路径
            
            save_image(im, b);
            printf("save %s successfully!\n",GetFilename(path));
#ifdef OPENCV
            cvNamedWindow("predictions", CV_WINDOW_NORMAL); 
            if(fullscreen){
                cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
            }
            show_image(im, "predictions");
            cvWaitKey(0);
            cvDestroyAllWindows();
#endif
        }
 
        free_image(im);
        free_image(sized);
        if (filename) break;
        }
      }
    }
}

 (2) 然后在前面添加*GetFilename(char *p)函数

#include"darknet.h"
#include<sys/stat.h>
#include<stdio.h>
#include<time.h>
#include<unistd.h>
#include<sys/types.h>

static int coco_ids[] = {1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90};
 
char *GetFilename(char *p)
{ 
    static char name[20]={""};
    char *q = strrchr(p,'/') + 1;
    strncpy(name,q,6);
    return name;
}

(3) 重新在darknet下make

(4) 执行下面命令开始测试 (把权重名字修改)

./darknet detector test cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -thresh 0.1

  YOLOv3默认阈值为0.25,-thresh 0是设置阈值,设为0的话可以显示出所有检测结果。

  运行完直接会让你输入 Enter Image Path:

 这里直接输入voc生成好的test文档的绝对路径就可以,里面包含了所有测试集的绝对路径。

3.使YOLOv3输出测试结果保存在txt文档中

./darknet detector valid cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -out detect_result.txt -thresh 0.1

4. 此次目标大赛txt文档格式为 图片id 类id 置信度 4个坐标,需要修改刚刚生成的txt

cat 123.txt | awk '{$7=$6;$6=$5;$5=$4;$4=$3;$3=$2;$2=1; print $0;}' > 123_chage.txt

5. 将生成的几个txt合并到一起

cat 1.txt 2.txt 3.txt > result.txt

6. 对合并到一起的txt按第一列(图片ID)的值进行排序,目的是使同一图的信息罗列在一起

sort -t $'\t' -k1 -n result.txt > 1result.txt # k1指的是第一列

7. 可以通过修改detector.c文件使得完成3-6步骤

  先修改validate_detector中的第二个if语句,直接复制替换

    if(0==strcmp(type, "coco")){
        if(!outfile) outfile = "coco_results";
        snprintf(buff, 1024, "%s/%s.json", prefix, outfile);
        fp = fopen(buff, "w");
        fprintf(fp, "[\n");
        coco = 1;
    } else if(0==strcmp(type, "imagenet")){
        if(!outfile) outfile = "imagenet-detection";
        snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
        fp = fopen(buff, "w");
        imagenet = 1;
        classes = 200;
    } else {
        if(!outfile) outfile = "comp4_det_test_";
        snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
        fp = fopen(buff, "w"); 
    }

  再修改validate_detector函数中的语句 print_detector_detections(fps, id, dets, nboxes, classes, w, h); 

  将fps修改为fp:print_detector_detections(fp, id, dets, nboxes, classes, w, h);

  再用下面代码替换 print_detector_detections 函数

void print_detector_detections(FILE *fp, char *id, detection *dets, int total, int classes, int w, int h)
{
    int i, j;
    int nums[4]={1,2,3,4}; #自己class有几类就相应的改掉,这里的1234对应的顺序和voc.name中一样
    for(i = 0; i < total; ++i){
        float xmin = dets[i].bbox.x - dets[i].bbox.w/2. + 1;
        float xmax = dets[i].bbox.x + dets[i].bbox.w/2. + 1;
        float ymin = dets[i].bbox.y - dets[i].bbox.h/2. + 1;
        float ymax = dets[i].bbox.y + dets[i].bbox.h/2. + 1;

        if (xmin < 1) xmin = 1;
        if (ymin < 1) ymin = 1;
        if (xmax > w) xmax = w;
        if (ymax > h) ymax = h;

        for(j = 0; j < classes; ++j){
            if (dets[i].prob[j]) fprintf(fp, "%s %d %f %f %f %f %f\n", id, nums[j], dets[i].prob[j],
                    xmin, ymin, xmax, ymax);
        }
    }
}

  然后重新make,再执行生成txt文档的语句,就可以把所有class的检测结果输出在一个txt文档中。

  PS:make之前记得删除原有的darknet可执行文件

三、处理离线比赛给的xml节点的代码

1. 将xml中的三类object分别替换为1 2 3

import os
import sys
try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
path = "/media/leequens/File/YOLO/test/xml/123" #文件夹路径
filelist = os.listdir(path) #读取该文件夹下的所有文件
for file in filelist: #循环每个文件
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    tree = ET.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
    for elem in tree.iter(tag='name'):  # 遍历树中的name节点
        if elem.text == '"scallop"':
            elem.text = '1'
        if elem.text == '"seaurchin"':
            elem.text = '2'
        if elem.text == '"seacucumber"':
            elem.text = '3'
    tree.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml') #修改完毕后写入另一个文件夹中

2. 将比赛官方给的xml中的frame改为自己重命名后的图片名

from xml.etree import ElementTree
import os
path = "/media/leequens/File/YOLO/test/xml/123"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
    node= xmldoc.find('./frame')  # 遍历树中的movie节点
    node.text=str(int(filename)).zfill(6) # 如果需要图片后缀 +'.jpg'
    xmldoc.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')


3. 修改xml中节点frame为filename

from xml.etree import ElementTree
import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/11"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/rebox/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    for child in root:
        if child.tag == 'frame':
            temp_node = 'filename' # 不懂为啥需要一个临时的变量,直接赋值就不对
            child.tag = temp_node
            break	
    xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')

4. 给xml增加图片长宽等信息

from xml.dom import minidom, Node
from xml.etree import ElementTree

import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    folder=ElementTree.Element('folder')
    root.insert(0,folder)
    folder.text='VOC2007'
    size = ElementTree.Element("size")
    root.insert(2,size)
    width =ElementTree.Element('width')
    width.text = '1920'
    size.insert(0,width)
    height =ElementTree.Element('height')
    height.text = '1080'
    size.insert(1,height)
    depth =ElementTree.Element('depth ')
    depth.text = '3'
    size.insert(2,depth )
    xmldoc.write('/media/leequens/File/YOLO/test/xml/perfectxml/'+str(int(filename)).zfill(6)+'.xml')


  数值自己要改,可以用下面代码查看所有图片长宽

# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image

# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/2018image'


# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
    if os.path.isdir(dir_path):
        for x in os.listdir(dir_path):
            get_img_list(os.path.join(dir_path, x))
    elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
        img_Lists.append(dir_path)

get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])

for img_item in img_Lists:
    filename = os.path.splitext(img_item)[0]
    im = Image.open(img_item)  #打开图片 为了记录图片的长宽数据
    img = os.path.split(img_item)[1].split('.')[0]
    width, height = im.size
    print(filename,width,height)

5. 给object加入difficult等节点

from xml.dom import minidom, Node
from xml.etree import ElementTree

import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/111"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    pose=ElementTree.Element('pose')
    pose.text='Unspecified'
    pose.tail='\n\t'
    truncated = ElementTree.Element('truncated')
    truncated.text='0'
    truncated.tail ='\n\t'
    difficult = ElementTree.Element('difficult')
    difficult.text = '0'
    difficult.tail ='\n\t'
    for node in xmldoc.iter(tag='object'):
    #node = root.find('./object')
    #objec=ElementTree.Element("object")
        node.insert(1,pose)
        node.insert(2,truncated)
        node.insert(3,difficult)
    xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')


6. 因为测试集是随机分配的,计算mAP时需要提取出测试集所对应的xml文件,测试集的名字都被存储到了test.txt文档里,所以需要编写一个程序把这些xml全部提取出来

import shutil
txt_file= open("111.txt","r",encoding="utf-8",errors="ignore")
while True:
    line = txt_file.readline() #表示一次读取一行
    line=line.strip('\n') # 这里要删除换行符,不然找不到对应文件,头疼
    if not line: #读到数据最后跳出,结束循环。数据的最后也就是读不到数据了,mystr为空的时候
        break
    oldname=u"/media/leequens/File/YOLO/test/xml/111/"+line+".xml" #旧目录
    newname=u"/media/leequens/File/YOLO/test/xml/"+line+".xml" # 新目录
    shutil.copyfile(oldname, newname)

./darknet detector demo cfg/voc.data yolov3-voc.cfg yolov3-voc_20000.weights "http://192.168.254.1:8090/?action=stream"