test class modify

Author: robinhood-jim

hadooptool/src/test/java/com/robin/test/TestCloudStorage.java

@@ -43,23 +43,19 @@ public void testWriteToMinio(){
         builder.addColumn("amount",Const.META_TYPE_INTEGER,null);
         builder.addColumn("type",Const.META_TYPE_INTEGER,null);
 
-        builder.resourceCfg(ResourceConst.PARQUETFILEFORMAT,ResourceConst.PARQUETSUPPORTFORMAT.PROTOBUF.getValue()).fileFormat(Const.FILEFORMATSTR.ARROW.getValue())
-                .resPath("tmp/bigdata3.arrow")
+        builder.resourceCfg(ResourceConst.PARQUETFILEFORMAT,ResourceConst.PARQUETSUPPORTFORMAT.PROTOBUF.getValue()).fileFormat(Const.FILEFORMATSTR.CSV.getValue())
+                .resPath("/tmp/bigdata3.csv.gz").resourceCfg("protocol",Const.VFS_PROTOCOL.FTP.getValue())
+                .resourceCfg("hostName","127.0.0.1").resourceCfg("userName","test").resourceCfg("password","test").protocol(Const.VFS_PROTOCOL.FTP.getValue()).fsType(Const.FILESYSTEM.VFS.getValue())
                 //.resourceCfg(ResourceConst.USEASYNCUPLOAD,"true")
                 .resourceCfg(ResourceConst.DEFAULTCACHEOFFHEAPSIZEKEY,1000*1000*6);
         ResourceBundle bundle=ResourceBundle.getBundle("minio");
         DataCollectionMeta colmeta=builder.build();
-        MinioFileSystemAccessor.Builder builder1=new MinioFileSystemAccessor.Builder();
+        /*MinioFileSystemAccessor.Builder builder1=new MinioFileSystemAccessor.Builder();
         MinioFileSystemAccessor accessor=builder1.accessKey(bundle.getString("minio.accessKey")).secretKey(bundle.getString("minio.secretKey")).endpoint(bundle.getString("minio.endpoint"))
-                .bucket("test").build();
-		/*QiniuFileSystemAccessor.Builder builder1=new QiniuFileSystemAccessor.Builder();
-		ResourceBundle bundle=ResourceBundle.getBundle("qiniu");
+                .bucket("test").build();*/
 
-		builder1.domain(bundle.getString("domain")).region(Region.autoRegion()).bucket(bundle.getString("bucket"))
-				.accessKey(bundle.getString("accessKey")).urlPrefix(bundle.getString("urlPrefix")).secretKey(bundle.getString("secretKey"));
-		QiniuFileSystemAccessor accessor=builder1.build();*/
 
-        try (AbstractFileWriter jwriter = (AbstractFileWriter) TextFileWriterFactory.getWriterByType(colmeta, accessor)){
+        try (AbstractFileWriter jwriter = (AbstractFileWriter) TextFileWriterFactory.getWriterByType(colmeta)){
             System.out.println(new Date());
             jwriter.beginWrite();
             Map<String, Object> recMap = new HashMap<>();
@@ -69,7 +65,7 @@ public void testWriteToMinio(){
                 priceMap.put(i,i*10.0);
             }
 
-            for (int i = 0; i < 50000; i++) {
+            for (int i = 0; i < 10000; i++) {
                 recMap.put("id", Long.valueOf(i));
                 recMap.put("name", StringUtils.generateRandomChar(32));
                 recMap.put("description", StringUtils.generateRandomChar(32));
@@ -98,9 +94,10 @@ public void testReadFromMinio(){
         builder.addColumn("type",Const.META_TYPE_INTEGER,null);
 
         builder.resourceCfg(ResourceConst.PARQUETFILEFORMAT,ResourceConst.PARQUETSUPPORTFORMAT.AVRO.getValue())
-                .resourceCfg(ResourceConst.STORAGEFILTERSQL,"select name,sno,type,price*amount as totalFee from test where price*amount>500 and sno<7 and name like 'A%'")
+                //.resourceCfg(ResourceConst.STORAGEFILTERSQL,"select name,sno,type,price*amount as totalFee from test where price*amount>500 and sno<7 and name like 'A%'")
+                .resourceCfg(ResourceConst.STORAGEFILTERSQL,"select type,sno,sum(price*amount) as totalFee from test where price*amount>500 group by sno,type having sum(price*amount)>100000.0")
                 .fileFormat(Const.FILEFORMATSTR.CSV.getValue()).tableName("test")
-                .resPath("tmp/bigdata3.CSV.gz");
+                .resPath("tmp/bigdata3.csv.gz");
         ResourceBundle bundle=ResourceBundle.getBundle("minio");
         DataCollectionMeta colmeta=builder.build();
         MinioFileSystemAccessor.Builder builder1=new MinioFileSystemAccessor.Builder();

hadooptool/src/test/java/com/robin/test/TestProtobufWriter.java

@@ -44,9 +44,9 @@ public void testInsert(){
             colmeta.setFileFormat(Const.FILESUFFIX_PROTOBUF);
 
             colmeta.setPath("d:/tmp/luoming/1.proto.gz");
-            AbstractFileSystemAccessor util= FileSystemAccessorFactory.getResourceAccessorByType(Const.FILESYSTEM.LOCAL.getValue());
+            AbstractFileSystemAccessor util= FileSystemAccessorFactory.getResourceAccessorByType(Const.FILESYSTEM.LOCAL.getValue(),colmeta);
 
-            final AbstractFileWriter jwriter= (AbstractFileWriter) TextFileWriterFactory.getOutputStreamByType(colmeta,util.getOutResourceByStream(colmeta,colmeta.getPath()));
+            final AbstractFileWriter jwriter= (AbstractFileWriter) TextFileWriterFactory.getOutputStreamByType(colmeta,util.getOutResourceByStream(colmeta.getPath()));
             jwriter.beginWrite();
             SimpleJdbcDao.executeOperationWithQuery(connection, "select id,cs_id,item_name,item_value from t_sys_code", null, false, new ResultSetOperationExtractor() {
                 @Override

hadooptool/src/test/java/com/robin/test/TestResourceGen.java

@@ -6,6 +6,7 @@
 import com.robin.core.base.datameta.DataBaseMetaFactory;
 import com.robin.core.base.datameta.DataBaseParam;
 import com.robin.core.base.util.Const;
+import com.robin.core.fileaccess.fs.FileSystemAccessorFactory;
 import com.robin.core.fileaccess.meta.DataCollectionMeta;
 import com.robin.core.fileaccess.writer.AbstractFileWriter;
 import com.robin.core.fileaccess.writer.TextFileWriterFactory;
@@ -51,15 +52,15 @@ public static void main(String[] args){
 			List<Map<String, Object>> list=SimpleJdbcDao.queryString(conn, "select config_name as name,config_value as value from t_hadoop_cluster_config where cluster_id=4");
 			conn=SimpleJdbcDao.getConnection(meta1);
 			List<Map<String, Object>> resultlist=SimpleJdbcDao.queryString(conn, "select info_id,url,title,content from shw_internet_info_dtl");
-			HdfsFileSystemAccessor util=new HdfsFileSystemAccessor();
+			HdfsFileSystemAccessor util= (HdfsFileSystemAccessor) FileSystemAccessorFactory.getResourceAccessorByType("hdfs");
 			Map<String, Object> hdfsparam=new HashMap<String, Object>();
 			for (Map<String, Object> tmap:list) {
 				hdfsparam.put(tmap.get("name").toString(), tmap.get("value"));
 			}
 			colmeta.setResourceCfgMap(hdfsparam);
 			colmeta.setPath("/testdata/test1.gz");
 			colmeta.setEncode("UTF-8");
-			pair=util.getOutResourceByWriter(colmeta, colmeta.getPath());
+			pair=util.getOutResourceByWriter(colmeta.getPath());
 			colmeta.setFileFormat(Const.FILETYPE_JSON);
 			AbstractFileWriter jwriter=(AbstractFileWriter) TextFileWriterFactory.getWriterByType(colmeta, pair.getKey());
 			System.out.println(new Date());