Thursday, January 16, 2020

Power Query M - Code for Options for reading CSV or ZIP files

=============================================================

1º A seguir, segue uma forma mais "artesanal" de se juntar vários CSV em um dataset, através do script:
(The following is a more "handcrafted" way to merge multiple CSVs into one dataset through the script)

let
    Source201907 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201907.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201906 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201906.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201905 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201905.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201904 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201904.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201903 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201903.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201902 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201902.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201901 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201901.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201812 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201812.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201811 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201811.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201810 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201810.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201809 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201809.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201808 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201808.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201807 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201807.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201806 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201806.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),
    Source201805 = Csv.Document(File.Contents("C:\Temp\datasets\csv\Sales201805.CSV"),[Delimiter=";", Encoding=1252, QuoteStyle=QuoteStyle.None]),

    TabelaSource = Table.Combine({#"Source201907",
                                  #"Source201906",
                                  #"Source201905",
                                  #"Source201904",
                                  #"Source201903",
                                  #"Source201902",
                                  #"Source201901",
                                  #"Source201812",
                                  #"Source201811",
                                  #"Source201810",
                                  #"Source201809",
                                  #"Source201808",
                                  #"Source201807",
                                  #"Source201806",
                                  #"Source201805"                               
                                  }),
    #"Renamed Columns" = Table.RenameColumns(TabelaSource,{{"Column1", "DtVenda"}, {"Column2", "Moeda"}, {"Column5", "CodSku"}, {"Column9", "Postos"}, {"Column16", "VlrVenda"}, {"Column19", "VlrVendaLiq"}, {"Column17", "VlrIVA"}, {"Column18", "QtdSKUs"}, {"Column4", "CodSAP"}, {"Column15", "DtHora"}, {"Column6", ".Canal"}, {"Column7", ".SetorAtiv"}, {"Column8", ".Empresa"}, {"Column21", ".Attibut1"}}),
    #"Replaced Value" = Table.ReplaceValue(#"Renamed Columns",".",",",Replacer.ReplaceText,{"VlrVendaLiq"}),
    #"Replaced Value1" = Table.ReplaceValue(#"Replaced Value",".",",",Replacer.ReplaceText,{"VlrIVA"}),
    #"Replaced Value2" = Table.ReplaceValue(#"Replaced Value1",".",",",Replacer.ReplaceText,{"QtdSKUs"}),
    #"Replaced Value3" = Table.ReplaceValue(#"Replaced Value2",".",",",Replacer.ReplaceText,{"VlrVenda"}),
    #"Changed Type" = Table.TransformColumnTypes(#"Replaced Value3",{{"VlrVenda", type number}, {"VlrIVA", type number}, {"QtdSKUs", type number}, {"VlrVendaLiq", type number}})
in
    #"Changed Type"

=============================================================

1º - Criar uma função chamada UnzipContents:
(Create a function called UnzipContents)

(ZIPFile) =>
let
    Header = BinaryFormat.Record([
        MiscHeader = BinaryFormat.Binary(14),
        BinarySize = BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger32, ByteOrder.LittleEndian),
        FileSize   = BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger32, ByteOrder.LittleEndian),
        FileNameLen= BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger16, ByteOrder.LittleEndian),
        ExtrasLen  = BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger16, ByteOrder.LittleEndian)
    ]),

    HeaderChoice = BinaryFormat.Choice(
        BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger32, ByteOrder.LittleEndian),
        each if _ <> 67324752             // not the IsValid number? then return a dummy formatter
            then BinaryFormat.Record([IsValid = false, Filename=null, Content=null])
            else BinaryFormat.Choice(
                    BinaryFormat.Binary(26),      // Header payload - 14+4+4+2+2
                    each BinaryFormat.Record([
                        IsValid  = true,
                        Filename = BinaryFormat.Text(Header(_)[FileNameLen]),
                        Extras   = BinaryFormat.Text(Header(_)[ExtrasLen]),
                        Content  = BinaryFormat.Transform(
                            BinaryFormat.Binary(Header(_)[BinarySize]),
                            (x) => try Binary.Buffer(Binary.Decompress(x, Compression.Deflate)) otherwise null
                        )
                        ]),
                        type binary                   // enable streaming
                )
    ),

    ZipFormat = BinaryFormat.List(HeaderChoice, each _[IsValid] = true),

    Entries = List.Transform(
        List.RemoveLastN( ZipFormat(ZIPFile), 1),
        (e) => [FileName = e[Filename], Content = e[Content] ]
    )
in
    Table.FromRecords(Entries)

2º Criar através de uma nova tabela (_Blank), através do script:
(Create through a new table (_Blank) through the script)

let
    Source = File.Contents("C:\Temp\datasets\csv\Sales2018.zip"),
    Files = UnzipContents(Source),
    #"Filtered Hidden Files1" = Table.SelectRows(Files, each [Attributes]?[Hidden]? <> true),
    #"Invoke Custom Function1" = Table.AddColumn(#"Filtered Hidden Files1", "Transform File", each #"Transform File"([Content])),
    #"Removed Other Columns1" = Table.SelectColumns(#"Invoke Custom Function1", {"Transform File"}),
    #"Expanded Table Column1" = Table.ExpandTableColumn(#"Removed Other Columns1", "Transform File", Table.ColumnNames(#"Transform File"(#"Sample File"))),
    #"Changed Type" = Table.TransformColumnTypes(#"Expanded Table Column1",{{"Column1", Int64.Type}, {"Column2", type text}, {"Column3", type text}, {"Column4", Int64.Type}, {"Column5", Int64.Type}, {"Column6", Int64.Type}, {"Column7", Int64.Type}, {"Column8", type text}, {"Column9", type text}, {"Column10", Int64.Type}, {"Column11", type text}, {"Column12", Int64.Type}, {"Column13", type text}, {"Column14", Int64.Type}, {"Column15", Int64.Type}, {"Column16", type number}, {"Column17", type number}, {"Column18", type number}, {"Column19", type number}, {"Column20", type text}, {"Column21", Int64.Type}})
in
    #"Changed Type"

Fonte:
http://sql10.blogspot.com/2016/06/reading-zip-files-in-powerquery-m.html

No comments:

Post a Comment

Power BI APP - Delete or Unpublish

 UNPUBLISH DELETE