--- a/TableData.st Wed Oct 30 22:29:01 2019 +0100
+++ b/TableData.st Tue Nov 05 15:45:01 2019 +0100
@@ -22,7 +22,9 @@
The category DWD (Deutscher Wetter Dienst / German weather service)
contains specific code to deal with historic weather data.
-
+
+ plot requires the r language to be installed (uses 'r' command).
+
[author:]
Claus Gittinger
@@ -37,6 +39,26 @@
[see also:]
"
+!
+
+examples
+"
+ self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv';
+ inspect.
+
+ ((self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv' separator:$, skip:1)
+ masie_meanPerMonthOfColumnNamed:'(0) Northern_Hemisphere')
+ plot
+
+ '/tmp/xxx.csv' asFilename writingFileDo:[:s |
+ ((self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv' separator:$, skip:1)
+ masie_meanPerMonthOfColumnNamed:'(0) Northern_Hemisphere')
+ writeCSVToStream:s
+ ]
+"
! !
!TableData class methodsFor:'instance creation'!
@@ -371,47 +393,47 @@
!TableData methodsFor:'analysis - DWD'!
-extractMonth:monthIndex
+dwd_extractMonth:monthIndex
"return a new table containing only rows for that month.
This is specific to DWD data"
- ^ self extractMonth:monthIndex columnName:'MESS_DATUM' format:'%4y%2m%2d'
+ ^ self dwd_extractMonth:monthIndex columnName:'MESS_DATUM' format:'%4y%2m%2d'
"
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19530101_20171231_04931.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
tableNameSuffix:'-Echterdingen';
plot
@@ -423,47 +445,47 @@
"Modified (comment): / 09-01-2019 / 18:30:09 / Claus Gittinger"
!
-extractMonth:monthIndex columnName:monthColumnName
+dwd_extractMonth:monthIndex columnName:monthColumnName
"return a new table containing only rows for that month.
This is specific to DWD data"
- ^ self extractMonth:monthIndex columnName:monthColumnName format:'%4y%2m%2d'
+ ^ self dwd_extractMonth:monthIndex columnName:monthColumnName format:'%4y%2m%2d'
"
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19530101_20171231_04931.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
tableNameSuffix:'-Echterdingen';
plot
@@ -475,7 +497,7 @@
"Modified (comment): / 09-01-2019 / 18:31:31 / Claus Gittinger"
!
-extractMonth:monthIndex columnName:monthColumnName format:dateFormat
+dwd_extractMonth:monthIndex columnName:monthColumnName format:dateFormat
"return a new table containing only rows for that month.
This is specific to DWD data"
@@ -494,45 +516,45 @@
"
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:1 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:2 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:8 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:9 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19530101_20171231_04931.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:10 columnName:'MESS_DATUM' format:'%4y%2m')
tableNameSuffix:'-Echterdingen';
plot
(((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19530101_20171231_04931.txt')
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:3 columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:3 columnName:'MESS_DATUM' format:'%4y%2m')
tableNameSuffix:'-Echterdingen';
plot
@@ -541,8 +563,8 @@
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19530101_20171231_04931.txt').
1 to:12 do:[:month |
((table
- meanPerMonthOfColumnNamed:'TXK')
- extractMonth:month columnName:'MESS_DATUM' format:'%4y%2m')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
+ dwd_extractMonth:month columnName:'MESS_DATUM' format:'%4y%2m')
tableNameSuffix:'-Echterdingen';
plot
]
@@ -553,14 +575,14 @@
"Created: / 09-01-2019 / 18:27:28 / Claus Gittinger"
!
-extractMonth:month day:day
+dwd_extractMonth:month day:day
"return a new table containing only rows for that day in month"
- ^ self extractMonth:month day:day columnName:'MESS_DATUM' format:'%4y%2m%2d'
+ ^ self dwd_extractMonth:month day:day columnName:'MESS_DATUM' format:'%4y%2m%2d'
"
((self fromFile:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- extractMonth:1 day:1)
+ dwd_extractMonth:1 day:1)
tableName:'1st Januar, Feldberg';
plot:{ #x -> 'MESS_DATUM' . #y -> { 'TXK' . 'TNK' }}
"
@@ -571,7 +593,7 @@
"Modified (comment): / 10-01-2019 / 16:20:48 / Claus Gittinger"
!
-extractMonth:month day:day columnName:monthColumnName format:dateFormat
+dwd_extractMonth:month day:day columnName:monthColumnName format:dateFormat
"return a new table containing only rows for that day in month"
|outRows dateColIdx|
@@ -587,7 +609,7 @@
"
((self fromFile:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- extractMonth:1 day:1 columnName:'MESS_DATUM' format:'%4y%2m%2d')
+ dwd_extractMonth:1 day:1 columnName:'MESS_DATUM' format:'%4y%2m%2d')
plot:{ #x -> 'MESS_DATUM' . #y -> { 'TXK' . 'TNK' }}
"
@@ -596,7 +618,7 @@
"Created: / 10-01-2019 / 16:14:29 / Claus Gittinger"
!
-meanPerMonthOfColumnNamed:colName
+dwd_meanPerMonthOfColumnNamed:colName
"return a new table containing the arithmetic mean per month of a column"
|outRows valColIdx dateColIdx prevYearAndMonth sum countDaysPerMonth|
@@ -634,7 +656,7 @@
"
((self new
readCSV:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerMonthOfColumnNamed:'TXK')
+ dwd_meanPerMonthOfColumnNamed:'TXK')
plot
"
@@ -645,7 +667,7 @@
"Modified (comment): / 09-01-2019 / 12:24:34 / Claus Gittinger"
!
-meanPerYearOfColumnNamed:colName
+dwd_meanPerYearOfColumnNamed:colName
"return a new table containing the arithmetic mean per year of a column"
|outRows valColIdx dateColIdx prevYear sum countDaysPerYear|
@@ -683,7 +705,7 @@
"
((self new
read:'/Users/cg/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt')
- meanPerYearOfColumnNamed:'TXK')
+ dwd_meanPerYearOfColumnNamed:'TXK')
plot
"
@@ -694,6 +716,61 @@
"Modified: / 06-01-2019 / 22:34:06 / Claus Gittinger"
! !
+!TableData methodsFor:'analysis - masie'!
+
+masie_meanPerMonthOfColumnNamed:colName
+ "return a new table containing the arithmetic mean per month of a column
+ of masie data"
+
+ |outRows valColIdx dateColIdx prevMonth prevYearAndMonth sum countDaysPerMonth|
+
+ dateColIdx := self indexOfColumnNamed:'yyyyddd'.
+ valColIdx := self indexOfColumnNamed:colName.
+
+ outRows := OrderedCollection new.
+ prevMonth := nil.
+ sum := countDaysPerMonth := 0.
+ rowData do:[:row |
+ | date rowsDate rowsYear month rowsDay yearAndMonth rowsYearAndDay val |
+
+ rowsDate := row at:dateColIdx.
+ "/ yyyyddd
+ rowsYear := Integer readFrom:(rowsDate copyTo:4).
+ rowsDay := Integer readFrom:(rowsDate copyFrom:5).
+ date := Date newDay:rowsDay year:rowsYear.
+ month := date month.
+ yearAndMonth := '%4d%02d' printfWith:date year with:date month.
+ prevMonth isNil ifTrue:[
+ prevMonth := month.
+ sum := countDaysPerMonth := 0.
+ ].
+ (month = prevMonth) ifTrue:[
+ val := Number readFrom:(row at:valColIdx).
+ sum := sum + val.
+ countDaysPerMonth := countDaysPerMonth + 1.
+ ] ifFalse:[
+ outRows add:{ prevYearAndMonth . (sum / countDaysPerMonth) }.
+ sum := countDaysPerMonth := 0.
+ prevYearAndMonth := yearAndMonth.
+ prevMonth := month.
+ ].
+ ].
+ ^ self class
+ rows:outRows columnNames:{ 'YearAndMonth' . 'Mean per month of ',colName }
+ tableName:('Mean Per Month of ',colName)
+
+ "
+ ((self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv' separator:$, skip:1)
+ masie_meanPerMonthOfColumnNamed:'(0) Northern_Hemisphere')
+ plot
+
+ "
+
+ "Created: / 06-01-2019 / 19:06:36 / Claus Gittinger"
+ "Modified (comment): / 09-01-2019 / 12:24:34 / Claus Gittinger"
+! !
+
!TableData methodsFor:'helpers'!
indexOfColumnNamed:name
@@ -781,7 +858,7 @@
outFile asFilename contents:''.
width := 1200.
- height := 400.
+ height := 800.
optionalSpec notNil ifTrue:[
spec := optionalSpec isDictionary
@@ -852,6 +929,14 @@
tmax tableName:'Feldberg'.
tmax plot:{ #x -> 'MESS_DATUM' . #y -> { 'TXK' . 'TNK' }}.
"
+ "
+ |data tmax|
+
+ data := self fromFile:'/Users/exept/Desktop/klima/monatswerte_KL_01050_19240101_20181231_hist/produkt_klima_monat_19240101_20181231_01050.txt'.
+ tmax := data withColumnsNamed:#('MESS_DATUM_BEGINN' 'MO_TT').
+ tmax tableName:'Feldberg'.
+ tmax plot:{ #x -> 'MESS_DATUM_BEGINN' . #y -> { 'MO_TT' }}.
+ "
"Created: / 06-01-2019 / 21:32:43 / Claus Gittinger"
"Modified: / 30-05-2019 / 11:28:42 / Claus Gittinger"
@@ -871,44 +956,6 @@
"Created: / 06-01-2019 / 15:59:47 / Claus Gittinger"
"Modified: / 06-01-2019 / 21:32:59 / Claus Gittinger"
-!
-
-writeCSVToStream:aStream
- "save myself as CSV onto a stream"
-
- columnNames
- do:[:nm |
- (nm includes:$;) ifTrue:[
- aStream nextPut:$"; nextPutAll:nm; nextPut:$".
- ] ifFalse:[
- aStream nextPutAll:nm
- ]]
- separatedBy:[aStream nextPut:$;].
- aStream cr.
- rowData do:[:row |
- row
- do:[:col |
- |s|
- s := col printString.
- (s includes:$;) ifTrue:[
- aStream nextPut:$"; nextPutAll:s; nextPut:$".
- ] ifFalse:[
- aStream nextPutAll:s
- ]
- ]
- separatedBy:[aStream nextPut:$;].
- aStream cr.
- ].
-
- "
- |data tmax|
-
- data := self fromFile:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt'.
- tmax := data withColumnsNamed:#('MESS_DATUM' 'TXK' 'TNK').
- String streamContents:[:s | tmax writeCSVToStream:s]
- "
-
- "Created: / 09-01-2019 / 17:48:25 / Claus Gittinger"
! !
!TableData methodsFor:'processing'!
@@ -1038,7 +1085,7 @@
!TableData methodsFor:'reading'!
readCSV:filename
- ^ self readCSV:filename separator:$;
+ ^ self readCSV:filename separator:$; skip:0
"
self new
@@ -1046,13 +1093,32 @@
inspect.
"
+ "
+ self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv';
+ inspect.
+ "
"Created: / 09-01-2019 / 12:07:27 / Claus Gittinger"
!
readCSV:filename separator:separatorCharacter
+ ^ self readCSV:filename separator:separatorCharacter skip:0
+
+ "
+ self new
+ readCSV:'/Users/cg/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt';
+ inspect.
+ "
+
+ "Created: / 09-01-2019 / 12:11:11 / Claus Gittinger"
+!
+
+readCSV:filename separator:separatorCharacter skip:numLinesToSkip
|headLine numCols dataLine ls word row rows|
filename asFilename readingFileDo:[:s |
+ numLinesToSkip timesRepeat:[s nextLine].
+
rows := OrderedCollection new.
headLine := s nextLine.
numCols := (headLine occurrencesOf:separatorCharacter)+1.
@@ -1080,12 +1146,69 @@
"Created: / 09-01-2019 / 12:11:11 / Claus Gittinger"
!
+readCSV:filename skip:numLinesToSkip
+ ^ self readCSV:filename separator:$; skip:numLinesToSkip
+
+ "
+ self new
+ readCSV:'/Users/cg/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt';
+ inspect.
+ "
+
+ "
+ self new
+ readCSV:'/Users/exept/Downloads/masie_4km_allyears_extent_sqkm.csv' skip:1;
+ inspect.
+ "
+ "Created: / 09-01-2019 / 12:07:27 / Claus Gittinger"
+!
+
readDemoCSV
self readCSV:'/Users/cg/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt'.
"Created: / 09-01-2019 / 12:07:52 / Claus Gittinger"
! !
+!TableData methodsFor:'writing'!
+
+writeCSVToStream:aStream
+ "save myself as CSV onto a stream"
+
+ columnNames
+ do:[:nm |
+ (nm includes:$;) ifTrue:[
+ aStream nextPut:$"; nextPutAll:nm; nextPut:$".
+ ] ifFalse:[
+ aStream nextPutAll:nm
+ ]]
+ separatedBy:[aStream nextPut:$;].
+ aStream cr.
+ rowData do:[:row |
+ row
+ do:[:col |
+ |s|
+ s := col printString.
+ (s includes:$;) ifTrue:[
+ aStream nextPut:$"; nextPutAll:s; nextPut:$".
+ ] ifFalse:[
+ aStream nextPutAll:s
+ ]
+ ]
+ separatedBy:[aStream nextPut:$;].
+ aStream cr.
+ ].
+
+ "
+ |data tmax|
+
+ data := self fromFile:'~/Documents/klima/data/DWD/daily/kl/historical/produkt_klima_tag_19450101_20171231_01346.txt'.
+ tmax := data withColumnsNamed:#('MESS_DATUM' 'TXK' 'TNK').
+ String streamContents:[:s | tmax writeCSVToStream:s]
+ "
+
+ "Created: / 09-01-2019 / 17:48:25 / Claus Gittinger"
+! !
+
!TableData class methodsFor:'documentation'!
version_CVS