欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识！
您现在的位置是： 首页

caffe自定义模型训练的方法

程序员文章站 2022-03-11 11:07:01
...

1.准备数据集

共分为5类:

animal

flowers

plane

house

guitars

下载并进行解压

2.数据集进行分类


# coding:utf-8

import numpy as np
import os
import shutil

'''
    数据集的分类
    测试集: 300张
    训练集: 500张
'''

root_caffe = 'D:/Development/caffe/caffe-master/caffe-master/'

type = ['train','test']
data_type = ['animal', 'flower', 'guitar', 'house', 'plane']

#创建文件夹
model_dir = 'models/my_models_recognition/data/'

#数据集存放的路径
database_path = 'H:/database/'

#创建一个模型的总文件夹
path = root_caffe + model_dir
if not os.path.exists(path):
    os.makedirs(path)

for i in range(2):
    if not os.path.exists(path + type[i]):
        os.makedirs(path + type[i])

    for j in range(5):
        if not os.path.exists(path + type[i] + '/' + data_type[j]):
            os.makedirs(path + type[i] + '/' + data_type[j])

        # 读取文件
        for root,dir,file in os.walk(database_path + data_type[j]):
            print(len(file))
            paths = []
            if(i == 0): # 训练集
                list = file[0:500]

                for k in range(500):
                    print(database_path + data_type[j] + '/'+ list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/'+ list[k]  , #图片的原始路径
                                    path + type[i] + '/' + data_type[j] + '/' + list[k]  ) #图片的caffe模型路径

            elif(i==1): # 测试集
                list = file[0:300]

                for k in range(300):
                    print(database_path + data_type[j] + '/' + list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/' + list[k],
                                    path + type[i] + '/' + data_type[j] + '/' + list[k])

3.标签文件的制作


# coding: utf-8

import os  


# In[2]:

#定义Caffe根目录
caffe_root = 'D:/Development/caffe/caffe-master/caffe-master/'

#制作训练标签数据
i = 0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/train.txt','wb') as train_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'+str(dir)): #遍历每一个文件夹中的文件 
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    train_txt.writelines(label)                   #写入标签文件中
                i+=1 #编号加1


#制作测试标签数据
i=0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/test.txt','wb') as test_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'+str(dir)): #遍历每一个文件夹中的文件
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    test_txt.writelines(label)                   #写入标签文件中
                i+=1#编号加1
                
print "成功生成文件列表"

caffe自定义模型训练的方法

4.数据转化，把图片转化为LMDB格式

caffe自定义模型训练的方法

convert_imageset_train.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%

SET GLOG_logtostderr=1
D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\train\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\train.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\train\

pause

convert_imageset_test.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%


D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\test\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\test.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\test\

pause

caffe自定义模型训练的方法

5.编写网络

用的是caffenet的网络，对此网络进行修改

caffe自定义模型训练的方法

train_val.prototxt:

name: "CaffeNet"
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
#  transform_param { #求均值方法1
#    mirror: true
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

#mean pixel / channel-wise mean instead of mean image
  transform_param { #求均值方法2
    crop_size: 227  #随机截取227*227
    mean_value: 104 #大量图片求的均值
    mean_value: 117
    mean_value: 123
    mirror: true
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/train"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
#  transform_param {
#    mirror: false
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

# mean pixel / channel-wise mean instead of mean image
  transform_param {
    crop_size: 227
    mean_value: 104
    mean_value: 117
    mean_value: 123
    mirror: false
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/test"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 96
    kernel_size: 11
    stride: 4
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm1"
  type: "LRN"
  bottom: "pool1"
  top: "norm1"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "norm1"
  top: "conv2"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 2
    kernel_size: 5
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm2"
  type: "LRN"
  bottom: "pool2"
  top: "norm2"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv3"
  type: "Convolution"
  bottom: "norm2"
  top: "conv3"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}
layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}
layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "pool5"
  top: "fc6"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc7"
  top: "fc7"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 5
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "accuracy"
  type: "Accuracy"
  bottom: "fc8"
  bottom: "label"
  top: "accuracy"
  include {
    phase: TEST
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "fc8"
  bottom: "label"
  top: "loss"
}

6.修改超参数文件

solver.prototxt:

net: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/train_val.prototxt"
test_iter: 30 #1000 /batch
test_interval: 200  #间隔数

base_lr: 0.01
lr_policy: "step"  #每隔stepsize步，对学习率进行改变
gamma: 0.1
stepsize: 1000

display: 100    #每100次显示一次
max_iter: 5000
momentum: 0.9
weight_decay: 0.0005
snapshot: 5000
snapshot_prefix: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/model"
solver_mode: CPU

7.编写train的bat文件

 %train训练数据%
 %超参数文件%

D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\caffe.exe train ^
-solver=D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/solver.prototxt

pause

这里是训练5000次 每1000次打印一次，我这里只是训练5000次之后，对文件进行保存:

caffe自定义模型训练的方法

8.进行测试

      8.1 编写网络文件

caffe自定义模型训练的方法

name: "CaffeNet"
layer {
  name: "data"
  type: "Input"
  top: "data"
  input_param { shape: { dim: 1 dim: 3 dim: 227 dim: 227 } }	 #dim:1 修改
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  convolution_param {
    num_output: 96
    kernel_size: 11
    stride: 4
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm1"
  type: "LRN"
  bottom: "pool1"
  top: "norm1"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "norm1"
  top: "conv2"
  convolution_param {
    num_output: 256
    pad: 2
    kernel_size: 5
    group: 2
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm2"
  type: "LRN"
  bottom: "pool2"
  top: "norm2"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv3"
  type: "Convolution"
  bottom: "norm2"
  top: "conv3"
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}
layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    group: 2
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}
layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    group: 2
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "pool5"
  top: "fc6"
  inner_product_param {
    num_output: 512  #修改，原因自己电脑的显存或cpu有限
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  inner_product_param {
    num_output: 512	  #修改，原因自己电脑的显存或cpu有限
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc7"
  top: "fc7"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  inner_product_param {
    num_output: 5   #修改类别
  }
}
layer {
  name: "prob"
  type: "Softmax"
  bottom: "fc8"
  top: "prob"
}

      8.2 找训练图片

               在数据集中每样寻找几张

caffe自定义模型训练的方法

  8.3编写python文件测试


import caffe
import numpy as np
import matplotlib.pyplot as plt
import os
import PIL
from PIL import Image
import sys


#定义Caffe根目录  这个需要根据自己的目录进行修改
caffe_root = 'D:/Development/caffe/caffe-master/caffe-master/'
#网络结构描述文件
deploy_file = caffe_root+'models/my_models_recognition/deploy.prototxt'
#训练好的模型
model_file = caffe_root+'models/my_models_recognition/model/model_iter_5000.caffemodel'

#gpu模式
#caffe.set_device(0)
caffe.set_mode_cpu()

#定义网络模型
net = caffe.Classifier(deploy_file, #调用deploy文件
                       model_file,  #调用模型文件
                       channel_swap=(2,1,0),  #caffe中图片是BGR格式，而原始格式是RGB，所以要转化
                       raw_scale=255,         #python中将图片存储为[0, 1]，而caffe中将图片存储为[0, 255]，所以需要一个转换
                       image_dims=(227, 227)) #输入模型的图片要是227*227的图片


#分类标签文件
imagenet_labels_filename = caffe_root +'models/my_models_recognition/labels/label.txt'
#载入分类标签文件
labels = np.loadtxt(imagenet_labels_filename, str)

#对目标路径中的图像，遍历并分类
for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/image/'): #循环每一个文件
    for file in files:
        #加载要分类的图片
        image_file = os.path.join(root,file)
        input_image = caffe.io.load_image(image_file)

        #打印图片路径及名称
        image_path = os.path.join(root,file)
        print(image_path)
        
        #显示图片
        img=Image.open(image_path)
        plt.imshow(img)
        plt.axis('off')
        plt.show()
        
        #预测图片类别
        prediction = net.predict([input_image])
        print 'predicted class:',prediction[0].argmax()

        # 输出概率最大的前5个预测结果
        top_k = prediction[0].argsort()[::-1]
        for node_id in top_k:     
            #获取分类名称
            human_string = labels[node_id]
            #获取该分类的置信度
            score = prediction[0][node_id]
            print('%s (score = %.5f)' % (human_string, score))

8.4 测试结果:

caffe自定义模型训练的方法

caffe自定义模型训练的方法

相关标签： caffe 自定义

上一篇： react-native---rn中的组件（TextInput）

下一篇： React Native组件（四）TextInput组件解析

推荐阅读