Linux shell命令统计某列去重后的值

需求：根据mac获取，mac对应的硬件版本和软件版本。在根据硬件版本统计mac的数量

$ head test_1.txt

00:07:67,ec2108,hwv010537p0000
00:07:63,z83,znv861010p1225
00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000
00:07:63,sbox8900hisi3716v3,zgv865012p0001
00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000
00:21:26,ec6108v9u_ca_sccdx,hwv218012p0000
00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000
00:07:63,stbhd-st7105-000,ztv70433089p0002
00:21:26,ec6108v9u_ca_sccdx,hwv215012p0000
00:21:26,ec6108v9u_ca_sccdx,hwv218012p0000

第二列去重

$ cat test_2.txt | awk -f ',' '{print $2}' | sort | uniq

ec2108
ec6108v9u_ca_sccdx
sbox8900hisi3716v3
stbhd-st7105-000
z83

将字符串转化为列表

import os
path = "c:\\users\\administrator\\desktop\\stb\\dd.txt"
read = open(path,"r")
list_ = []
for line in read:
  line_ = line.strip()
  list_.append(line_)
print(list_)

根据厂商和硬件版本获取对应的软件版本

方法一：

get gather-012-20180109/_search
{
 "size": 0, 
 "aggs": {
  "1": {
   "terms": {
    "field": "dev_hard_ver",
    "size": 200
   },
   "aggs": {
    "2": {
     "terms": {
      "field": "soft_ver",
      "size": 1
     }
    }
   }
  }
 }
}

方法二：

import os
from elasticsearch import elasticsearch
path = "c:\\users\\administrator\\desktop\\stb\\fact_stb.txt"
write_file = open(path,"a+")
es = elasticsearch()
def get_data(dev_hard_ver,dev_vendor_name):
  index_ = "gather-012-20180109"
  try:
    rs = es.search(index=index_,body={
      "size": 1,
       "query": {
        "bool": {
         "must": [
          {"term": {
           "dev_hard_ver": {
            "value": dev_hard_ver
           }
          }
         },
         {
          "term": {
           "dev_vendor_name": {
            "value": dev_vendor_name
           }
          }
         }
         ]
        }
       },
       "_source": ["soft_ver","dev_vendor_name","dev_hard_ver"]
    })
    return rs
  except:
    return -1
dev_hard_ver=["02l0b61701mtas0001", "0x209", "0x441", "0x485", "0xe0a", "0xe17", "16060102l0b61701mtas0001", "1828", "1903", "2035", "5202", "99007002l0b6170635100791", "9900b002l0b61701iho3000t", "990104010004102300000001", "99010402l0b6083100000001", "990501010003170908a5c800", "a7.32.64.a0", "b860av2.1", "bjds2z-52", "chep3000sc", "chi619112", "chiho3000", "chiho3000a", "chiho3000jx2", "chiho33", "chiho3300a", "chiho3300ad", "cub-androidstb", "ec2106v1", "ec2108", "ec6108v9_pub_gdydx", "ec6108v9u_ca_sccdx", "ec6108v9u_pub_hbjdx", "ec6108v9u_pub_sccdx", "g2-40_2544", "hg510i", "hg650-tdswh", "hg650-tdswh-11", "ht670-v", "hy_v1.0", "hy107", "hy112", "hy127", "hy2021", "hy34e71c02l0b6v003hg510p04", "hy34e71c02l0b6v3.0hg510p10", "hy44170102l0b6v003hg510p04", "hy9900e0013798kk0120170001", "hy990104010004102300000001", "hyc37.e", "hychep920sc", "hychep921sc", "hyee", "hyhg510iv1.0", "hyhg510pv3.0", "hyty_h1.00.00", "hyv10-androidigw-068", "hyyix-g210v1.1", "ibhn-androidstb-068", "ibhr-androidstbga-069", "ibhu-androidstb-068", "ibhx-androidstbga-068", "ibhx-androidstbga-069", "iptv-s63-v1.1", "iptv-s65-v1.1", "iptv-s65-v1.2", "itv628 hd", "kb1.100.01h", "m8043v02", "mr222-dsgh-19", "mx_ca_sccdx", "s80-e-r1.01.01", "sbox8900hisi3716v3", "sbox8900hisi3716v386501007", "stbab-androidstb-068", "stbab-androidstbga-069", "stbhd-aml8626x-000", "stbhd-st7105-000", "stbhd-st7105-128-256-000", "stbho-aml8626x-000", "tgh0001", "ty_a1.00.00", "ty_h1.00.00", "v1", "v10-androidigw-068", "v2.1", "z82", "z83", "z85e", "zxb700v5c(ibhl-000-000)", "zxb860a(ibhr-androidstb-068)", "zxb860a(ibhr-androidstbga-068)"]
dev_vendor_name=["scty", "changhong", "jiuzhou", "jiuzhou", "hisense", "hisense", "scty", "skyworth", "skyworth", "skyworth", "huawei", "zte", "changhong", "skyworth", "skyworth", "zhaoneng", "utstarcom", "zte", "fiberhome", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "youhua", "huawei", "huawei", "huawei", "huawei", "huawei", "huawei", "yinhe", "utstarcom", "fiberhome", "fiberhome", "zhaoneng", "scty", "hisense", "hisense", "hisense", "skyworth", "yhtc", "yhtc", "yhtc", "fiberhome", "skyworth", "huawei", "changhong", "changhong", "hisense", "utstarcom", "yhtc", "scty", "zte", "ccs", "zte", "zte", "zte", "zte", "zte", "chinagci", "chinagci", "chinagci", "changhong", "30kt", "huawei", "fiberhome", "huawei", "chinagci", "sunniwell", "sunniwell", "zte", "zte", "zte", "zte", "zte", "zte", "yihekeji", "scty", "scty", "zte", "zte", "fiberhome", "zhaoneng", "zhaoneng", "zhaoneng", "zte", "zte", "zte"]
for i in range(len(dev_vendor_name)):
  rs = get_data(dev_hard_ver[i],dev_vendor_name[i])
  if rs!=-1:
    for hit in rs['hits']['hits']:
      soft_ver=hit['_source']['soft_ver']
      if soft_ver is none:
        soft_ver='none'
      write_file.write(soft_ver+"\n")
    else:
      write_file.write("-1" + "\n")
write_file.close()

总结

以上就是这篇文章的全部内容了，希望本文的内容对大家的学习或者工作具有一定的参考学习价值，谢谢大家对的支持。如果你想了解更多相关内容请查看下面相关链接