基于tensorflow的图片数据处理

这里给出一些处理图片的方法，主要是应用于tensorflow的多线程队列读取图片文件，划分train数据集和val数据集，这里借助于sklearn函数随机划分。下面主要是处理一个文件夹下有多个文件夹，同时提取多个文件夹的图片，文件的名作为lablelName

这里划分的数据集是按照文件下的文件夹下内容划分的，这样划分不平衡分布数据更加合理。

def logs_set(string):
    f = open('label_categories.txt','a+')
    f.write(string+'\n')
    f.close()
def onehot(length,index):
    if length > index:
        code = []
        for i in range(length):
            code.append(0)
        code[index] = 1
        return code
    else :
        return None


def get_image_path(dir,w = width,h=height,classes=classes):
    if os.path.isdir(dir):
        codes = []
        image_path = []
        labels = []
        final = []
        test = []
        index = 0
        for class_name in os.listdir(dir):
            now_path = dir + '/' + class_name
            code = []
            if os.path.isdir(now_path):
                code = onehot(classes,index)
                #logs_set(class_name)
                base= []
                lab = []
                for image_name in os.listdir(now_path):
                    if image_name.endswith('.jpg') and not image_name.startswith('.'):
                        filename = now_path + '/' + image_name
                        image_path.append(filename)
                        codes.append(code)
                        labels.append(class_name)
                        base.append(filename)
                        lab.append(class_name)
                final.append(base)
                test.append(lab)
                index += 1
        return final,test

这里的label_categories文件是label name这里是为了进行方便编码处理。

实现数据shuffle处理，随机性

''
shuffle the data
'''
def shuffle_data(images,labels):
    temp = np.array([images,labels])
    temp = temp.transpose()
    np.random.shuffle(temp)
    imgs = list(temp[:,0])
    labs = list(temp[:,1])
    return imgs , labs

数据划分4/5，自己按照实际要求来进行分配：这里要quote 文件from sklearn.model_selection import train_test_split

def get_labels_split_data(train,label):
    train_img = []
    train_la = []
    test_img = []
    test_la = []
    for i in range(len(train)):
        train_list = train[i]
        label_list = label[i]
        train_x,test_x,train_y,test_y=trainsplit_data(train_list,label_list,split_rate=0.2)
        train_img.append(train_x)
        train_la.append(train_y)
        test_img.append(test_x)
        test_la.append(test_y)
    return train_img,train_la,test_img,test_la

def get_labels_split_data(train,label):
    train_img = []
    train_la = []
    test_img = []
    test_la = []
    for i in range(len(train)):
        train_list = train[i]
        label_list = label[i]
        train_x,test_x,train_y,test_y=trainsplit_data(train_list,label_list,split_rate=0.2)
        train_img.append(train_x)
        train_la.append(train_y)
        test_img.append(test_x)
        test_la.append(test_y)
    return train_img,train_la,test_img,test_la

数据文件夹名的连接、label编码实现：

def ReadLabels(name='label_categories.txt'):
    f = open(name,'r+')
    lable = []
    while True:
        line = f.readline()
        if not line :
            break
        name = line.split('\n')[0]
        lable.append(name)
    f.close()
    return lable
def getIndexByName(name,filename='label_categories.txt'):
    labels = ReadLabels(filename)
    index = 0
    for i in range(len(labels)):
        if labels[i] == name :
            index = i
            break
    return index
def get_totat_list_path(train_x,train_y):
    train = []
    labels = []
    for i in range(len(train_x)):
        train = train + train_x[i]
        labels = labels + train_y [i]

    return train,labels
def get_index_by_Classname(labels):
    lab = []
    for i in range(len(labels)):
        index = getIndexByName(labels[i])
        lab.append(index)
    return lab

以下提供数据读取处理

一种是制作层TFCord文件保存，训练时就读取TFCord文件：

例如

def train_data_create(data_save,train_x,train_y,w = width,h=height):
    writer = tf.python_io.TFRecordWriter(data_save)
    for i in range(len(train_x)):
        filename = train_x[i]
        index = train_y[i]
        '''
        img = cv2.imread(filename)
        top, bottom, left, right = getPaddingSize(img)
                        # 将图片放大， 扩充图片边缘部分
        img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value = [0, 0, 0])
        img = cv2.resize(img, (h, w))
        img_raw = img.tobytes()
        '''
        img = New_test(filename)#这是图片数据的文件
        img_raw = img.tobytes()
        example = tf.train.Example(features=tf.train.Features(feature={
                    "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[index])),
                    'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
               }))
        writer.write(example.SerializeToString())
    writer.close()

测试例如：

if __name__ == '__main__':    

    path,labels = get_image_path(load_dir)

    train_x , train_y , test_x ,test_y = get_labels_split_data(path,labels)
    train_x , train_y = shuffle_data(train_x,train_y)
    test_x , test_y = shuffle_data(test_x,test_y)
    train_x , train_y = shuffle_data(train_x,train_y)
    test_x , test_y = shuffle_data(test_x,test_y)
    print(len(train_x),len(train_y),train_x,test_y)

    train_x , train_y = get_totat_list_path(train_x,train_y)
    test_x , test_y = get_totat_list_path(test_x,test_y)
    print(len(train_y),len(test_y))
    test_y = get_index_by_Classname(test_y)
    train_y = get_index_by_Classname(train_y)
    print(train_y,test_y,len(train_y),len(test_y))
    train_data_create(train_save_dir, train_x, train_y, w = width, h=height)
    train_data_create(test_save_dir, test_x, test_y, w = width, h=height)

文件的读取如下：

def loadFile(filename):
    filename_queue = tf.train.string_input_producer([filename])
    reader = tf.TFRecordReader()
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        features={
            'label': tf.FixedLenFeature([], tf.int64),
            'img_raw': tf.FixedLenFeature([], tf.string)
        }
    )
    img = tf.decode_raw(features['img_raw'],tf.uint8)
    img = tf.cast(img, tf.float32)
    img = tf.reshape(img, [width,height,3], name=None)
    label = tf.cast(features['label'], tf.int32)
    #label = ChangeOneHot(label.eval())
    return img,label

def get_train_batch(batch_size,dir=train_save_dir):
    img , lable = loadFile(dir)
    capacity = 9200
    min_after = int(9200*0.4)
    image_batch , label_batch=tf.train.shuffle_batch([img,lable],batch_size=batch_size,num_threads=5,capacity=capacity,min_after_dequeue=min_after)
    return image_batch,label_batch
def get_test_batch(batch_size ,dir= test_save_dir):
    img , lable = loadFile(dir)
    capacity = 2000
    min_after = int(2000*0.4)
    image_batch , label_batch=tf.train.shuffle_batch([img,lable],batch_size=batch_size,num_threads=5,capacity=capacity,min_after_dequeue=min_after)
    return image_batch,label_batch
def onehot(length,index):
        code = []
        for i in range(length):
            code.append(0)
        code[index] = 1
        return code
def ChangeOneHot(labels):
    codes = []
    for i in range(len(labels)):
        code = onehot(classes, labels[i])
        codes.append(code)
    return codes

以上的数据处理自己是用来处理category256数据集的，希望可以减少数据处理的时间。自己也是在学习Deep Learning 。该数据集是一个很好的数据集。

至于在上面有一个category265数据集，图片处理我没有给出大家可以自己处理，因为有BGR训练的、有RGB训练的，这样就看大家对于图片的处理了，建议大家可以看看大佬的论文再去进行训练。how to train effeciently 、明白训练每一层的含义参数，而不是去直接调用别人包。

另一种是：这下面是我自己做COCO数据集做的处理训练，改成上面的读取非常简单，改动一下代码就可以了，自己懒没有给大家该，我是先制作的TFcord文件放到云服务器上去计算的。

def get_val_batch(batch_size,setBox=False):
    #train_x , test_x , train_y, test_y = get_total_data()
    image_batch=None
    lable_batch=None
    image , id ,annotinons ,box=  ReadJsonTrainFile(file_val_dir,3)
    label = MultiOneHot(id)
    images = tf.cast(image,tf.string)
    lables = tf.cast(label,tf.int32)
    input_queue = tf.train.slice_input_producer([images,lables])
    image_content = tf.read_file(input_queue[0])
    labels = input_queue[1]
    if setBox == False :
        final_image = []
        for i in range(len(image)):
            final_image.append(getRGB(image[i]))
        image = tf.cast(final_image,tf.float32)
        capa = 10000
        after = 4000
        image_batch , lable_batch = tf.train.shuffle_batch([image,labels],batch_size=batch_size,num_threads=8,capacity=capa,min_after_dequeue=after)
    #return image_batch,lable_batch
    else :
        final_image = []
        for i in range(len(image)):
            if box[i]:
                final_image.append(getImage(image[i],box[i]))
            else:
                final_image.append(getRGB(image[i]))
        image = tf.cast(final_image,tf.float32)
        capa = 10000
        after = 4000
        image_batch , lable_batch = tf.train.shuffle_batch([image,labels],batch_size=batch_size,num_threads=8,capacity=capa,min_after_dequeue=after)
    return image_batch,lable_batch