1.爬取指定关键字图片

'''
爬取指定关键字图片
'''
import re  # 正则表达式，解析网页
import requests  # 请求网页
import traceback
import os


def dowmloadPic(html, keyword, startNum):
    headers = {'user-agent': 'Mozilla/5.0'}  # 浏览器伪装，因为有的网站会反爬虫，通过该headers可以伪装成浏览器访问，否则user-agent中的代理信息为python
    pic_url = re.findall('"objURL":"(.*?)",', html, re.S)  # 找到符合正则规则的目标网站
    num = len(pic_url)
    i = startNum
    subroot = root + '/' + word
    # txtpath = subroot + '/download_detail.txt'

    print('找到关键词:' + keyword + '的图片，现在开始下载图片...')

    for each in pic_url:
        a = '第' + str(i + 1) + '张图片，图片地址:' + str(each) + '\n'
        b = '正在下载' + a
        print(b)
        path = subroot + '/' + str(i + 1)
        try:
            if not os.path.exists(subroot):
                os.mkdir(subroot)
            if not os.path.exists(path):
                pic = requests.get(each, headers=headers, timeout=10)
                with open(path + '.jpg', 'wb') as f:
                    f.write(pic.content)
                    f.close()
                # with open(txtpath, 'a') as f:
                #     f.write(a)
                #     f.close()

        except:
            continue
            # traceback.print_exc()
            # print('【错误】当前图片无法下载')
            # os.remove(pic.content)
            # pass
            # continue
        i += 1

    return i


if __name__ == '__main__':

    headers = {'user-agent': 'Mozilla/5.0'}
    # words = ['人拉行李箱', '行李箱和人', '人托行李箱']
    # words = ['黄色行李箱','黄色包包','黄色双肩包','黄色腰包','黄色手提包','黄色单肩包','黄色手提袋']
    words = ['人拉灰色行李箱','人背灰色包包','人背灰色双肩包','人背灰色腰包','人拿灰色手提包','人背灰色单肩包','人拿灰色手提袋']
    # words = ['粉色行李箱','粉色包包','粉色双肩包','粉色腰包','粉色手提包','粉色单肩包','粉色手提袋']
    # words = [ ‘蓝色行李箱','棕色包包','棕色双肩包','棕色腰包','棕色手提包','棕色单肩包','棕色手提袋']
    # words = ['绿色行李箱','绿色包包','绿色双肩包','绿色腰包','绿色手提包','绿色单肩包','绿色手提袋']
    # words = ['紫色行李箱','紫色包包','紫色双肩包','紫色腰包','紫色手提包','紫色单肩包','紫色手提袋']
    # words = ['棕色行李箱','棕色包包','棕色双肩包','棕色腰包','棕色手提包','棕色单肩包','棕色手提袋']
    # words = ['花色行李箱','花色包包','花色双肩包','花色腰包','花色手提包','花色单肩包','花色手提袋']
    # words = ['青色行李箱','青色包包','青色双肩包','青色腰包','青色手提包','青色单肩包','青色手提袋']
    # words = ['白色行李箱','白色包包','白色双肩包','白色腰包','白色手提包','白色单肩包','白色手提袋']
    # words为一个列表，可以自动保存多个关键字的图片
    root = './download_images'
    for word in words:
        root = root + word + '&'
    if not os.path.exists(root):
        os.mkdir(root)
    for word in words:
        lastNum = 0
        # word = input("Input key word: ")
        if word.strip() == "exit":
            break
        pageId = 0
        # 此处的参数为需爬取的页数，设置为30页
        for i in range(15):              #获取10*60张图
            url = 'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word=' + word + "&pn=" + str(
                pageId) + "&gsm=?&ct=&ic=0&lm=-1&width=0&height=0"
            pageId += 20
            html = requests.get(url, headers=headers)
            # print(html.text) #打印网页源码，相当于在网页中右键查看源码内容
            lastNum = dowmloadPic(html.text, word, lastNum, )  # 本条语句执行一次获取60张图

类似生成如下文件夹，每个文件夹都会有显示

2、每个文件夹进行合并成一个大的文件夹

import os
import shutil

#目标文件夹，此处为绝对路径，也可以是相对路径
determination =r'G:\pachong\1white'
if not os.path.exists(determination):
    os.makedirs(determination)

#源文件夹路径，根目录
path = r'G:\pachong\white'
#根目录下的所有一级目录，以列表形式赋给first_dir
first_dir = os.listdir(path)
#遍历每一个一级目录
for first in first_dir:
    #一级目录绝对路径
    dir = path + '/' + str(first)
    #得到一级目录下的二级目录
    # second_dir = os.listdir(dir)

    #遍历每一个二级目录
    # for second in second_dir:
    #     # 二级目录绝对路径
    #     source = dir + '/' + str(second)
        # 二级目录绝对路径下所有图片
    imgs = os.listdir(dir)

    for img in imgs:
        source_img = dir + '/' + str(img)
        deter = determination + '/' + str(first) + '_'  + '_' + str(img)
        shutil.copyfile(source_img, deter)

3.合并后进行筛选、转化三通道RGB

###############################################2.转通道、重命名、删小图等
import os.path
from PIL import Image
from PIL import ImageFile
MAGES = True
import cv2
from tqdm import tqdm
from PIL import Image
import os
ImageFile.LOAD_TRUNCATED_IMAGES = True
import glob

path = r"F:\1213bag\all_kind\111colour_all\bag_colour_all_v3\666\white"
files = os.listdir(path)


# print(files)
b = 0
i = 0
a = 0
for file in files:
    original = path + os.sep + files[b]
    new = path + os.sep + "whiteo" + str(b+1) + ".jpg"                   ###改颜色
    os.rename(original,new)
    b +=1

for pic in tqdm(glob.glob("F:/1213bag/all_kind/111colour_all/bag_colour_all_v3/666/white/*.jpg")):   ########## 改地址
    # # # basename = os.path.basename(image_name)
    # before_name = os.path.splitext(pic)[0]
    # txt_name = os.path.splitext(before_name)[0] + ".txt"
    # txt_name = os.path.join(save_path,txt_name)
    # f = open(txt_name, "w").
    # if pic.endwith('.txt'):
    #     os.remove(pic)

    try:
        img = Image.open(pic)
        # if img is None:
        #     os.remove(os.path.join(path, pic))
        # img.close()
        # print(pic)
        # print(img.getbands())  # ('P',) 这种是有彩色的，而L是没有彩色的
        # print(img.size)
        # Img = np.array(img)
        # a = np.unique(Img)
        # print(a)    #看像素值
        if len(img.getbands()) != 3:
              img = img.convert("RGB")
              pic_new = os.path.join(pic)
              img.save(pic_new)
              a +=1
        i += 1
        size = img.size
        w = size[0]  # 宽度
        h = size[1]  # 高度

        if w<100 or h<100:
            img.close()
            os.remove(pic)  #去除小图，爬虫下在下来的小图会有问题，必须img.close() 否则会报错，这张图正在使用进程
        # if w == 640:
        #     if h == 480:
        #         c += 1


    except:
        print('read image failed已删除!')
        # img = Image.open(pic)
        # img.close()
        # break
        os.remove(pic)




print('图像image的总数量: ', i)
print('总图像不是RGB的数量：', a)
# print('总图像是640*480的数量：', c)

4、去除重复图像

import shutil
import numpy as np
from PIL import Image
import os


def 比较图片大小(dir_image1, dir_image2):
    with open(dir_image1, "rb") as f1:
        size1 = len(f1.read())
    with open(dir_image2, "rb") as f2:
        size2 = len(f2.read())
    if (size1 == size2):
        result = "大小相同"
    else:
        result = "大小不同"
    return result


def 比较图片尺寸(dir_image1, dir_image2):
    image1 = Image.open(dir_image1)
    image2 = Image.open(dir_image2)
    if (image1.size == image2.size):
        result = "尺寸相同"
    else:
        result = "尺寸不同"
    return result


def 比较图片内容(dir_image1, dir_image2):
    image1 = np.array(Image.open(dir_image1))
    image2 = np.array(Image.open(dir_image2))
    if (np.array_equal(image1, image2)):
        result = "内容相同"
    else:
        result = "内容不同"
    return result


def 比较两张图片是否相同(dir_image1, dir_image2):
    # 比较两张图片是否相同
    # 第一步：比较大小是否相同
    # 第二步：比较长和宽是否相同
    # 第三步：比较每个像素是否相同
    # 如果前一步不相同，则两张图片必不相同
    result = "两张图不同"
    大小 = 比较图片大小(dir_image1, dir_image2)
    if (大小 == "大小相同"):
        尺寸 = 比较图片尺寸(dir_image1, dir_image2)
        if (尺寸 == "尺寸相同"):
            内容 = 比较图片内容(dir_image1, dir_image2)     
            if (内容 == "内容相同"):
                result = "两张图相同"
    return result


if __name__ == '__main__':

    load_path = r'F:\1213bag\all_kind\111colour_all\bag_colour_all_v3\666\white'  # 要去重的文件夹
    save_path = r'G:\pachong\13411sf'  # 空文件夹，用于存储检测到的重复的照片
    os.makedirs(save_path, exist_ok=True)

    # 获取图片列表 file_map，字典{文件路径filename : 文件大小image_size}
    file_map = {}
    image_size = 0
    # 遍历filePath下的文件、文件夹（包括子目录）
    for parent, dirnames, filenames in os.walk(load_path):
        # for dirname in dirnames:
        # print('parent is %s, dirname is %s' % (parent, dirname))
        for filename in filenames:
            # print('parent is %s, filename is %s' % (parent, filename))
            # print('the full name of the file is %s' % os.path.join(parent, filename))
            image_size = os.path.getsize(os.path.join(parent, filename))
            file_map.setdefault(os.path.join(parent, filename), image_size)

    # 获取的图片列表按 文件大小image_size 排序
    file_map = sorted(file_map.items(), key=lambda d: d[1], reverse=False)
    file_list = []
    for filename, image_size in file_map:
        file_list.append(filename)

    # 取出重复的图片
    file_repeat = []
    for currIndex, filename in enumerate(file_list):
        dir_image1 = file_list[currIndex]
        dir_image2 = file_list[currIndex + 1]
        result = 比较两张图片是否相同(dir_image1, dir_image2)
        if (result == "两张图相同"):
            file_repeat.append(file_list[currIndex + 1])
            print("\n相同的图片：", file_list[currIndex], file_list[currIndex + 1])
        else:
            print('\n不同的图片：', file_list[currIndex], file_list[currIndex + 1])
        currIndex += 1
        if currIndex >= len(file_list) - 1:
            break

    # 将重复的图片移动到新的文件夹，实现对原文件夹降重
    for image in file_repeat:
        shutil.move(image, save_path)
        print("正在移除重复照片：", image)

处理前会有重复图片和一些小图：


还有一些下载失败无法显示的图，点开无法显示
经过以上一系列处理之后，则如下图所示：