taov-erp / 8zyun-data-analysis

Download as
Browse Code »

Commit 8515ce3a5dedcc1831914f519b75b7485554f89e

Authored by unknown
1 parent 34ac7ab3
Exists in master

download and analysis file

Showing 13 changed files with 301 additions and 27 deletions   Show diff stats
build.gradle
  Diff comments   View file @8515ce3
... ... @@ -46,7 +46,7 @@ dependencies {
46 46  
47 47 "com.xuxueli:xxl-job-core:2.1.0",
48 48 "com.taover:com-taover-repository:2.1.22",
49   - "com.taover:com-taover-easyexcel:2.2.14",
  49 + "com.taover:com-taover-easyexcel:2.2.16",
50 50 "com.taover:com-taover-util:1.1.112",
51 51 "cn.hutool:hutool-all:5.2.4",
52 52 "com.taover:com-taover-codegenerate:1.2.11",
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelAnnoParser.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,45 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.lang.reflect.Constructor;
  4 +import java.lang.reflect.Field;
  5 +import java.util.ArrayList;
  6 +import java.util.List;
  7 +import java.util.Map.Entry;
  8 +
  9 +public class ExcelAnnoParser<D> implements ExcelParser<D>{
  10 + public static final String FIELD_NAME_ORIGIN_DATA = "originData";
  11 +
  12 + @Override
  13 + public List<D> parse(List<List<Object>> fieldData, Class<D> beanClass) throws Exception {
  14 + //读取文件
  15 + if(fieldData.isEmpty()) {
  16 + return new ArrayList<D>();
  17 + }
  18 +
  19 + //表头解析
  20 + ExcelHeaderParser headerParser = new ExcelHeaderAnnoParser();
  21 + ExcelHeaderResult headerParserResult = headerParser.parse(fieldData.get(0), beanClass);
  22 +
  23 + //数据注入
  24 + Constructor<D> cons = beanClass.getDeclaredConstructor();
  25 + cons.setAccessible(true);
  26 + List<D> data = new ArrayList<D>(fieldData.size());
  27 + for(int i=1; i<fieldData.size(); ++i) {
  28 + D item = cons.newInstance();
  29 + for(Entry<Field, Integer> reflect : headerParserResult.getHeaderNameMap().entrySet()) {
  30 + Field field = reflect.getKey();
  31 + Integer dataIndex = reflect.getValue();
  32 + if(field == null || dataIndex == null || dataIndex >= fieldData.size()) {
  33 + continue;
  34 + }
  35 + field.set(item, fieldData.get(i).get(dataIndex));
  36 + }
  37 + for(Field reflect: headerParserResult.getOriginDataFieldList()) {
  38 + reflect.set(item, fieldData.get(i));
  39 + }
  40 + data.add(item);
  41 + }
  42 +
  43 + return data;
  44 + }
  45 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelHeaderAnnoParser.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,53 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.lang.reflect.Field;
  4 +import java.util.ArrayList;
  5 +import java.util.HashMap;
  6 +import java.util.List;
  7 +import java.util.Map;
  8 +
  9 +public class ExcelHeaderAnnoParser implements ExcelHeaderParser {
  10 + @Override
  11 + public ExcelHeaderResult parse(List<Object> data, Class headerBeanClass) {
  12 + Map<Field, Integer> fieldNameToIndex = new HashMap<Field, Integer>();
  13 + List<Field> originDataFieldList = new ArrayList<Field>();
  14 +
  15 + if(data == null || data.isEmpty()) {
  16 + return new ExcelHeaderResult();
  17 + }
  18 +
  19 + Class currClass = headerBeanClass;
  20 + while(currClass != null
  21 + && !currClass.getSimpleName().equals("Object")) {
  22 + Field[] fields = currClass.getDeclaredFields();
  23 + for(int i=0; i<fields.length; ++i) {
  24 + Field item = fields[i];
  25 + item.setAccessible(true);
  26 +
  27 + //原始数据
  28 + ExcelOriginData dataAnno = item.getDeclaredAnnotation(ExcelOriginData.class);
  29 + if(dataAnno != null) {
  30 + originDataFieldList.add(item);
  31 + continue;
  32 + }
  33 +
  34 + //表头列
  35 + ExcelHeaderName anno = item.getDeclaredAnnotation(ExcelHeaderName.class);
  36 + if(anno == null || "".equals(anno.value().trim())) {
  37 + continue;
  38 + }
  39 + for(int j=0; j<data.size(); ++j) {
  40 + Object tempJ = data.get(j);
  41 + if(tempJ == null) {
  42 + continue;
  43 + }
  44 + if(anno.value().equals(tempJ.toString())) {
  45 + fieldNameToIndex.put(item, j);
  46 + }
  47 + }
  48 + }
  49 + currClass = headerBeanClass.getSuperclass();
  50 + }
  51 + return new ExcelHeaderResult(fieldNameToIndex, originDataFieldList);
  52 + }
  53 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelHeaderName.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,14 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.lang.annotation.Documented;
  4 +import java.lang.annotation.ElementType;
  5 +import java.lang.annotation.Retention;
  6 +import java.lang.annotation.RetentionPolicy;
  7 +import java.lang.annotation.Target;
  8 +
  9 +@Retention(RetentionPolicy.RUNTIME)
  10 +@Target(ElementType.FIELD)
  11 +@Documented
  12 +public @interface ExcelHeaderName {
  13 + String value();
  14 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelHeaderParser.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,7 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.util.List;
  4 +
  5 +interface ExcelHeaderParser {
  6 + ExcelHeaderResult parse(List<Object> data, Class headerBeanClass);
  7 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelHeaderResult.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,35 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.lang.reflect.Field;
  4 +import java.util.ArrayList;
  5 +import java.util.HashMap;
  6 +import java.util.List;
  7 +import java.util.Map;
  8 +
  9 +public class ExcelHeaderResult {
  10 + Map<Field, Integer> headerNameMap;
  11 + List<Field> originDataFieldList;
  12 +
  13 + public ExcelHeaderResult() {
  14 + this.headerNameMap = new HashMap<Field, Integer>();
  15 + this.originDataFieldList = new ArrayList<Field>();
  16 + }
  17 +
  18 + public ExcelHeaderResult(Map<Field, Integer> headerNameMap, List<Field> originDataFieldList) {
  19 + this.headerNameMap = headerNameMap;
  20 + this.originDataFieldList = originDataFieldList;
  21 + }
  22 +
  23 + public Map<Field, Integer> getHeaderNameMap() {
  24 + return headerNameMap;
  25 + }
  26 + public void setHeaderNameMap(Map<Field, Integer> headerNameMap) {
  27 + this.headerNameMap = headerNameMap;
  28 + }
  29 + public List<Field> getOriginDataFieldList() {
  30 + return originDataFieldList;
  31 + }
  32 + public void setOriginDataFieldList(List<Field> originDataFieldList) {
  33 + this.originDataFieldList = originDataFieldList;
  34 + }
  35 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelOriginData.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,13 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.lang.annotation.Documented;
  4 +import java.lang.annotation.ElementType;
  5 +import java.lang.annotation.Retention;
  6 +import java.lang.annotation.RetentionPolicy;
  7 +import java.lang.annotation.Target;
  8 +
  9 +@Retention(RetentionPolicy.RUNTIME)
  10 +@Target(ElementType.FIELD)
  11 +@Documented
  12 +public @interface ExcelOriginData {
  13 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/common/exceldeal/ExcelParser.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,7 @@
  1 +package com.taover.bazhuayun.analysis.common.exceldeal;
  2 +
  3 +import java.util.List;
  4 +
  5 +public interface ExcelParser<D> {
  6 + List<D> parse(List<List<Object>> fieldData, Class<D> beanClass) throws Exception;
  7 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/script/goodscollect/GoodsDataFileParser.java
  Diff comments   View file @8515ce3
... ... @@ -31,24 +31,34 @@ public class GoodsDataFileParser {
31 31  
32 32 public void parse(GroupInfoExcelData groupInfoData, File[] listFiles) throws Exception {
33 33 for(int i=0; i<listFiles.length; ++i) {
34   - File item = listFiles[i];
35   -
36   - //获取发件组/发件人
37   - String fileName = item.getName().replaceFirst(this.fileNameIgnorePreffix, "");
38   - GroupInfoExcelDataRow groupInfo = this.groupInfoData.findByFileName(fileName);
39   -
40   - //读取Excel文件
41   - List<List<Object>> data = ExcelUtil.readExcelSheet(item, false);
42   - if(data.isEmpty()) {
43   - continue;
  34 + try {
  35 + this.parseFile(listFiles[i]);
  36 + }catch(Exception e) {
  37 + e.printStackTrace();
44 38 }
45   -
46   - //写入set
47   - GoodsInfoExcelDataHeader header = new GoodsInfoExcelDataHeader(this.goodsNamePatternArr, this.goodsSkuNameParrternArr, data.get(0));
48   - for(int j=1; j<data.size(); ++j) {
49   - this.rowSet.add(new GoodsInfoRow(groupInfo, header, data.get(j)));
50   - }
51   - }
  39 + }
  40 + }
  41 +
  42 + private void parseFile(File item) throws Exception {
  43 + //获取发件组/发件人
  44 + String fileName = item.getName().replaceFirst(this.fileNameIgnorePreffix, "");
  45 + GroupInfoExcelDataRow groupInfo = this.groupInfoData.findByFileName(fileName);
  46 +
  47 + //读取Excel文件
  48 + List<List<Object>> data = ExcelUtil.readExcelSheet(item, false);
  49 + if(data.isEmpty()) {
  50 + return;
  51 + }
  52 +
  53 + //写入set
  54 + GoodsInfoExcelDataHeader header = new GoodsInfoExcelDataHeader(this.goodsNamePatternArr, this.goodsSkuNameParrternArr, data.get(0));
  55 + if(!header.available()) {
  56 + this.logStrBuffer.append("=================文件名称["+item.getName()+"]==================\n");
  57 + this.logStrBuffer.append("没找到商品列或规格列\n");
  58 + }
  59 + for(int j=1; j<data.size(); ++j) {
  60 + this.rowSet.add(new GoodsInfoRow(groupInfo, header, data.get(j)));
  61 + }
52 62 }
53 63  
54 64 public List<GoodsInfoRow> getResult() {
... ...
src/main/java/com/taover/bazhuayun/analysis/script/goodscollect/GoodsInfoExcelDataHeader.java
  Diff comments   View file @8515ce3
... ... @@ -25,6 +25,14 @@ public class GoodsInfoExcelDataHeader {
25 25 }
26 26 }
27 27  
  28 + public boolean available() {
  29 + if(this.goodsNameIndex != null || this.goodsSkuNameIndex != null) {
  30 + return true;
  31 + }else {
  32 + return false;
  33 + }
  34 + }
  35 +
28 36 public Integer getGoodsNameIndex() {
29 37 return goodsNameIndex;
30 38 }
... ...
src/main/java/com/taover/bazhuayun/analysis/script/goodscollect/Main.java
  Diff comments   View file @8515ce3
... ... @@ -19,10 +19,10 @@ public class Main {
19 19 }
20 20  
21 21 public static void analysisGoodsName() throws Exception {
22   - String[] goodsNamePatternArr = new String[]{"商品名称","商品标题","货品名称","商品名称","商品信息","品名","商品型号","商品名","商品简称"};
  22 + String[] goodsNamePatternArr = new String[]{"商品名称","商品标题","货品名称","商品名称","商品信息","品名","商品型号","商品名","商品简称","产品名称"};
23 23 String[] goodsSkuNameParrternArr = new String[]{"商品规格名","规格","商品规格","规格名称","商品型号"};
24   - String dataDirPath = "C:\\Users\\Administrator\\Desktop\\悟空家数据核对-lff";
25   - String groupInfoFilePath = dataDirPath + File.separator + "groupinfo.csv";
  24 + String dataDirPath = "C:\\Users\\Administrator\\Desktop\\悟空家21-25";
  25 + String groupInfoFilePath = dataDirPath + File.separator + "悟空家-仓库群文件-21-25.xlsx";
26 26 String goodsDataFileDirPath = dataDirPath + File.separator + "data";
27 27 String logFilePath = dataDirPath + File.separator + "dealinfo.log";
28 28 String resultDataFilePath = dataDirPath + File.separator + "result.xls";
... ... @@ -34,7 +34,7 @@ public class Main {
34 34 //读取文件列表，交由解析器处理
35 35 GoodsDataFileParser parser = new GoodsDataFileParser(goodsNamePatternArr, goodsSkuNameParrternArr, groupInfoData);
36 36 parser.setFileNameIgnorePreffix("【异常物流信息】-");
37   - parser.parse(groupInfoData, new File(goodsDataFileDirPath).listFiles());
  37 + parser.parse(groupInfoData, new File(goodsDataFileDirPath).listFiles());
38 38  
39 39 //结果写入结果文件
40 40 ExcelUtil.getWriter(new File(resultDataFilePath)).write(transListBeanTo2List(parser.getResult())).flush();
... ...
src/main/java/com/taover/bazhuayun/analysis/script/orderdownload/GroupInfoExcelDataBean.java 0 → 100644
  Diff comments   View file @8515ce3
... ... @@ -0,0 +1,55 @@
  1 +package com.taover.bazhuayun.analysis.script.orderdownload;
  2 +
  3 +import com.taover.bazhuayun.analysis.common.exceldeal.ExcelHeaderName;
  4 +
  5 +public class GroupInfoExcelDataBean {
  6 + @ExcelHeaderName("群名称")
  7 + private String groupName;
  8 + @ExcelHeaderName("room_wxid")
  9 + private String roomWxid;
  10 + @ExcelHeaderName("创建时间")
  11 + private String createdAt;
  12 + @ExcelHeaderName("发送人昵称")
  13 + private String senderNickname;
  14 + @ExcelHeaderName("文件路径")
  15 + private String ossUrl;
  16 + @ExcelHeaderName("仓库名称")
  17 + private String wareName;
  18 +
  19 + public String getGroupName() {
  20 + return groupName;
  21 + }
  22 + public void setGroupName(String groupName) {
  23 + this.groupName = groupName;
  24 + }
  25 + public String getRoomWxid() {
  26 + return roomWxid;
  27 + }
  28 + public void setRoomWxid(String roomWxid) {
  29 + this.roomWxid = roomWxid;
  30 + }
  31 + public String getCreatedAt() {
  32 + return createdAt;
  33 + }
  34 + public void setCreatedAt(String createdAt) {
  35 + this.createdAt = createdAt;
  36 + }
  37 + public String getSenderNickname() {
  38 + return senderNickname;
  39 + }
  40 + public void setSenderNickname(String senderNickname) {
  41 + this.senderNickname = senderNickname;
  42 + }
  43 + public String getOssUrl() {
  44 + return ossUrl;
  45 + }
  46 + public void setOssUrl(String ossUrl) {
  47 + this.ossUrl = ossUrl;
  48 + }
  49 + public String getWareName() {
  50 + return wareName;
  51 + }
  52 + public void setWareName(String wareName) {
  53 + this.wareName = wareName;
  54 + }
  55 +}
... ...
src/main/java/com/taover/bazhuayun/analysis/script/orderdownload/Main.java
  Diff comments   View file @8515ce3
1 1 package com.taover.bazhuayun.analysis.script.orderdownload;
2 2  
  3 +import java.io.File;
  4 +import java.util.List;
  5 +
  6 +import com.taover.bazhuayun.analysis.common.exceldeal.ExcelAnnoParser;
  7 +import com.taover.bazhuayun.analysis.common.exceldeal.ExcelParser;
  8 +import com.taover.bazhuayun.analysis.util.ExcelUtil;
  9 +import com.taover.util.UtilHttpByOkHttp;
  10 +
  11 +import cn.hutool.core.io.FileUtil;
  12 +
3 13 public class Main {
4   - public static void main(String[] args) {
5   - try {
6   - //时间区间：21号凌晨 25号凌晨
7   - } catch (Exception e) {
8   - e.printStackTrace();
  14 + public static final String ossUrlPreffix = "https://8zyun-oss.oss-cn-beijing.aliyuncs.com";
  15 +
  16 + public static void main(String[] args) throws Exception {
  17 + String dirPath = "C:\\Users\\Administrator\\Desktop\\悟空家21-25";
  18 + String urlFileName = "悟空家-仓库群文件-21-25.xlsx";
  19 + String dataDirName = "data";
  20 +
  21 + List<List<Object>> fieldData = ExcelUtil.readExcelSheet(new File(dirPath+File.separator+urlFileName), false);
  22 + ExcelParser<GroupInfoExcelDataBean> parser = new ExcelAnnoParser<GroupInfoExcelDataBean>();
  23 + List<GroupInfoExcelDataBean> beanData = parser.parse(fieldData, GroupInfoExcelDataBean.class);
  24 + for(GroupInfoExcelDataBean item: beanData) {
  25 + try {
  26 + downloadDatafile(item, dirPath+File.separator+dataDirName);
  27 + }catch (Exception e) {
  28 + e.printStackTrace();
  29 + }
9 30 }
10 31 }
  32 +
  33 + private static void downloadDatafile(GroupInfoExcelDataBean item, String dataDir) throws Exception {
  34 + byte[] byteData = UtilHttpByOkHttp.downloadFile(ossUrlPreffix+"/"+item.getOssUrl(), null);
  35 + String fileName = item.getOssUrl().substring(item.getOssUrl().lastIndexOf("/")+1);
  36 + FileUtil.writeBytes(byteData, dataDir+File.separator+fileName);
  37 + }
11 38 }
... ...