dnl PSPP - a program for statistical analysis.
dnl Copyright (C) 2017 Free Software Foundation, Inc.
dnl
dnl This program is free software: you can redistribute it and/or modify
dnl it under the terms of the GNU General Public License as published by
dnl the Free Software Foundation, either version 3 of the License, or
dnl (at your option) any later version.
dnl
dnl This program is distributed in the hope that it will be useful,
dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
dnl GNU General Public License for more details.
dnl
dnl You should have received a copy of the GNU General Public License
dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
dnl
AT_BANNER([GET DATA /TYPE=TXT])

dnl These tests exercise features of GET DATA /TYPE=TXT that
dnl it has in common with DATA LIST, using tests drawn from
dnl DATA LIST.

AT_SETUP([GET DATA /TYPE=TXT with explicit delimiters])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file=inline /delimiters="|X"
 /variables=A f7.2 B f7.2 C f7.2 D f7.2.
begin data.
1|23X45|2.03
2X22|34|23|
3|34|34X34
end data.

list.
])
AT_CHECK([pspp -o pspp.csv get-data.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
A,B,C,D
1.00,23.00,45.00,2.03
2.00,22.00,34.00,23.00
3.00,34.00,34.00,34.00
])
AT_CLEANUP

AT_SETUP([GET DATA /TYPE=TXT with FIRSTCASE])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file=inline /delimiters=', ' /delcase=variables 4
 /firstcase=2 /variables=A f7.2 B f7.2 C f7.2 D f7.2.
begin data.
# This record is ignored.
,1,2,3
,4,,5
6
7,

8 9
0,1,,,

,,,,

2

3
4
5
end data.
list.
])
AT_CHECK([pspp -o pspp.csv get-data.sps], [0], [ignore])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
A,B,C,D
.  ,1.00,2.00,3.00
.  ,4.00,.  ,5.00
6.00,7.00,.  ,8.00
9.00,.00,1.00,.  @&t@
.  ,.  ,.  ,.  @&t@
.  ,.  ,.  ,2.00
.  ,3.00,4.00,5.00
])
AT_CLEANUP

AT_SETUP([GET DATA /TYPE=TXT with FIRSTCASE and tab delimiter])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file=inline /delimiters='\t' /delcase=variables 4
 /firstcase=3 /variables=A f7.2 B f7.2 C f7.2 D f7.2.
begin data.
# These records
# are skipped.
1	2	3	4
1	2	3	4	@&t@
1	2		4
1	2		4	@&t@
1		3	4
1		3	4	@&t@
1			4
1			4	@&t@
	2	3	4
	2	3	4	@&t@
	2		4
	2		4	@&t@
		3	4
		3	4	@&t@
			4
			4	@&t@
end data.
list.
])
AT_CHECK([pspp -o pspp.csv get-data.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
A,B,C,D
1.00,2.00,3.00,4.00
1.00,2.00,3.00,4.00
1.00,2.00,.  ,4.00
1.00,2.00,.  ,4.00
1.00,.  ,3.00,4.00
1.00,.  ,3.00,4.00
1.00,.  ,.  ,4.00
1.00,.  ,.  ,4.00
.  ,2.00,3.00,4.00
.  ,2.00,3.00,4.00
.  ,2.00,.  ,4.00
.  ,2.00,.  ,4.00
.  ,.  ,3.00,4.00
.  ,.  ,3.00,4.00
.  ,.  ,.  ,4.00
.  ,.  ,.  ,4.00
])
AT_CLEANUP

AT_SETUP([GET DATA /TYPE=TXT with multiple records per case])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file=inline /arrangement=fixed /fixcase=3 /variables=
	/1 start 0-19 adate8
	/2 end 0-19 adate
	/3 count 0-2 f.
begin data.
07-22-2007
10-06-2007
321
07-14-1789
08-26-1789
4
01-01-1972
12-31-1999
682
end data.
list.
])
AT_CHECK([pspp -o pspp.csv get-data.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
start,end,count
07/22/07,10/06/2007,321
********,08/26/1789,4
01/01/72,12/31/1999,682
])
AT_CLEANUP

AT_SETUP([GET DATA /TYPE=TXT with empty trailing record])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file=inline /arrangement=fixed /fixcase=2 /variables=
	/1 x 0 f
           y 1 f.
begin data.
12

34

56

78

90

end data.
list.
])
AT_CHECK([pspp -o pspp.csv get-data.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
x,y
1,2
3,4
5,6
7,8
9,0
])
AT_CLEANUP

dnl This test is a copy of an example given in the manual
dnl in doc/files.texi.
AT_SETUP([GET DATA /TYPE=TXT password example])
AT_DATA([passwd.data], [dnl
root:$1$nyeSP5gD$pDq/:0:0:,,,:/root:/bin/bash
blp:$1$BrP/pFg4$g7OG:1000:1000:Ben Pfaff,,,:/home/blp:/bin/bash
john:$1$JBuq/Fioq$g4A:1001:1001:John Darrington,,,:/home/john:/bin/bash
jhs:$1$D3li4hPL$88X1:1002:1002:Jason Stover,,,:/home/jhs:/bin/csh
])
AT_DATA([passwd.sps], [dnl
GET DATA /TYPE=TXT /FILE='passwd.data' /DELIMITERS=':'
        /VARIABLES=username A20
                   password A40
                   uid F10
                   gid F10
                   gecos A40
                   home A40
                   shell A40.
LIST.
])
AT_CHECK([pspp -o pspp.csv passwd.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
username,password,uid,gid,gecos,home,shell
root,$1$nyeSP5gD$pDq/,0,0,",,,",/root,/bin/bash
blp,$1$BrP/pFg4$g7OG,1000,1000,"Ben Pfaff,,,",/home/blp,/bin/bash
john,$1$JBuq/Fioq$g4A,1001,1001,"John Darrington,,,",/home/john,/bin/bash
jhs,$1$D3li4hPL$88X1,1002,1002,"Jason Stover,,,",/home/jhs,/bin/csh
])
AT_CLEANUP

dnl This test is a copy of an example given in the manual
dnl in doc/files.texi.
AT_SETUP([GET DATA /TYPE=TXT cars example])
AT_DATA([cars.data], [dnl
model   year    mileage price   type    age
Civic   2002    29883   15900   Si      2
Civic   2003    13415   15900   EX      1
Civic   1992    107000  3800    n/a     12
Accord  2002    26613   17900   EX      1
])
AT_DATA([cars.sps], [dnl
GET DATA /TYPE=TXT /FILE='cars.data' /DELIMITERS=' ' /FIRSTCASE=2
        /VARIABLES=model A8
                   year F4
                   mileage F6
                   price F5
                   type A4
                   age F2.
LIST.

GET DATA /TYPE=TXT /FILE='cars.data' /ARRANGEMENT=FIXED /FIRSTCASE=2
        /VARIABLES=model 0-7 A
                   year 8-15 F
                   mileage 16-23 F
                   price 24-31 F
                   type 32-39 A
                   age 40-47 F.
LIST.
])
AT_CHECK([pspp -o pspp.csv cars.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
model,year,mileage,price,type,age
Civic,2002,29883,15900,Si,2
Civic,2003,13415,15900,EX,1
Civic,1992,107000,3800,n/a,12
Accord,2002,26613,17900,EX,1

Table: Data List
model,year,mileage,price,type,age
Civic,2002,29883,15900,Si,2
Civic,2003,13415,15900,EX,1
Civic,1992,107000,3800,n/a,12
Accord,2002,26613,17900,EX,1
])
AT_CLEANUP

dnl This test is a copy of an example given in the manual
dnl in doc/files.texi.
AT_SETUP([GET DATA /TYPE=TXT pets example])
AT_DATA([pets.data], [dnl
'Pet''s Name', "Age", "Color", "Date Received", "Price", "Height", "Type"
, (Years), , , (Dollars), ,
"Rover", 4.5, Brown, "12 Feb 2004", 80, '1''4"', "Dog"
"Charlie", , Gold, "5 Apr 2007", 12.3, "3""", "Fish"
"Molly", 2, Black, "12 Dec 2006", 25, '5"', "Cat"
"Gilly", , White, "10 Apr 2007", 10, "3""", "Guinea Pig"
])
AT_DATA([pets.sps], [dnl
GET DATA /TYPE=TXT /FILE='pets.data' /DELIMITERS=', ' /QUALIFIER='''"'
        /FIRSTCASE=3
        /VARIABLES=name A10
                   age F3.1
                   color A5
                   received EDATE10
                   price F5.2
                   height a5
                   type a10.
LIST.
])
AT_CHECK([pspp -o pspp.csv pets.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
name,age,color,received,price,height,type
Rover,4.5,Brown,12.02.2004,80.00,"1'4""",Dog
Charlie,. ,Gold,05.04.2007,12.30,"3""",Fish
Molly,2.0,Black,12.12.2006,25.00,"5""",Cat
Gilly,. ,White,10.04.2007,10.00,"3""",Guinea Pig
])
AT_CLEANUP
dnl " (fixes Emacs highlighting)

AT_SETUP([GET DATA /TYPE=TXT with IMPORTCASE])
AT_CHECK([$PYTHON3 > test.data -c '
for i in range(1, 101):
    print("%02d" % i)
'])
AT_DATA([get-data.sps], [dnl
get data /type=txt /file='test.data' /importcase=first 10 /variables x f8.0.
get data /type=txt /file='test.data' /importcase=percent 1 /variables x f8.0.
get data /type=txt /file='test.data' /importcase=percent 35 /variables x f8.0.
get data /type=txt /file='test.data' /importcase=percent 95 /variables x f8.0.
get data /type=txt /file='test.data' /importcase=percent 100 /variables x f8.0.
])
AT_CHECK([pspp -O format=csv get-data.sps], [0], [dnl
"get-data.sps:1.39-1.57: warning: GET DATA: Ignoring obsolete IMPORTCASES subcommand.  (N OF CASES or SAMPLE may be used to substitute.).
    1 | get data /type=txt /file='test.data' /importcase=first 10 /variables x f8.0.
      |                                       ^~~~~~~~~~~~~~~~~~~"

"get-data.sps:2.39-2.58: warning: GET DATA: Ignoring obsolete IMPORTCASES subcommand.  (N OF CASES or SAMPLE may be used to substitute.).
    2 | get data /type=txt /file='test.data' /importcase=percent 1 /variables x f8.0.
      |                                       ^~~~~~~~~~~~~~~~~~~~"

"get-data.sps:3.39-3.59: warning: GET DATA: Ignoring obsolete IMPORTCASES subcommand.  (N OF CASES or SAMPLE may be used to substitute.).
    3 | get data /type=txt /file='test.data' /importcase=percent 35 /variables x f8.0.
      |                                       ^~~~~~~~~~~~~~~~~~~~~"

"get-data.sps:4.39-4.59: warning: GET DATA: Ignoring obsolete IMPORTCASES subcommand.  (N OF CASES or SAMPLE may be used to substitute.).
    4 | get data /type=txt /file='test.data' /importcase=percent 95 /variables x f8.0.
      |                                       ^~~~~~~~~~~~~~~~~~~~~"

"get-data.sps:5.39-5.60: warning: GET DATA: Ignoring obsolete IMPORTCASES subcommand.  (N OF CASES or SAMPLE may be used to substitute.).
    5 | get data /type=txt /file='test.data' /importcase=percent 100 /variables x f8.0.
      |                                       ^~~~~~~~~~~~~~~~~~~~~~"
])
AT_CLEANUP

AT_SETUP([GET DATA /TYPE=TXT with ENCODING subcommand])
AT_CHECK([i18n-test supports_encodings UTF-8 ISO-8859-1])
AT_DATA([get-data.sps], [dnl
set locale='utf-8'
get data /type=txt /file='data.txt' /encoding='iso-8859-1'
  /delimiters="," /variables=s a8.
list.
])
printf '\351' > data.txt	# é in ISO-8859-1.
AT_CHECK([pspp -o pspp.csv get-data.sps])
AT_CHECK([cat pspp.csv], [0], [dnl
Table: Data List
s
é
])
AT_CLEANUP


AT_SETUP([GET DATA /TYPE= truncated])

AT_DATA([x.sps], [dnl
GET DATA /TYPE=
.
])

AT_CHECK([pspp -o pspp.csv x.sps], [1], [ignore])

AT_CLEANUP



AT_SETUP([GET DATA /TYPE=txt bug])


AT_DATA([thing.txt], [dnl
foo, title, last
1, this, 1
2, that, 2
3, other, 3
])

AT_DATA([x.sps], [dnl
GET DATA
  /TYPE=TXT
  /FILE="thing.txt"
  /ARRANGEMENT=DELIMITED
  /DELCASE=LINE
  /FIRSTCASE=2
  /DELIMITERS=","
  /VARIABLES=foo F1.0
    title A8
    last F2.0.

list.
])

AT_CHECK([pspp -O format=csv x.sps], [0], [dnl
Table: Data List
foo,title,last
1,this,1
2,that,2
3,other,3
])

AT_CLEANUP



AT_SETUP([GET DATA /TYPE=txt another bug])

AT_DATA([crash.sps], [dnl
get data /type=txt /file=inline /variables=C f7.2 D f7>2.
begin data.
3 2
4 2
5 2
end data.
])

AT_CHECK([pspp -O format=csv crash.sps], [1], [ignore])

AT_CLEANUP






