diff -Nru xmltv-0.6.1/Changes xmltv-0.6.3/Changes --- xmltv-0.6.1/Changes 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/Changes 2020-09-07 15:02:53.000000000 +0000 @@ -1,3 +1,32 @@ +0.6.3 2020-08-22 + + - tv_grab_ch_search: disable deprecated grabber + +0.6.2 2020-08-21 + + - tv_grab_dotmedia: disable deprecated grabber + - tv_grab_se_tvzon: disable deprecated grabber + + - tv_grab_dtv_la: disable broken grabber + - tv_grab_il: disable broken grabber + - tv_grab_pt_meo: disable broken grabber + - tv_grab_se_swedb: disable broken grabber + + * XMLTV.pm: update handling of reading from STDIN due to + XML::Parser adopting 3-arg open + * tv_grab_ch_search: handle upstream cookies + * tv_grab_eu_epgdata: various fixes and improvements + * tv_grab_fi: various fixes and improvements + * tv_grab_fr: update grabber due to upstream changes + * tv_grab_huro: use https source site URLs + * tv_grab_it: fix overlapping/duplicate programmes + * tv_grab_na_dd: use https source site URLs + * tv_grab_na_dtv: various fixes and improvements + * tv_grab_pt_vodafone: various fixes and improvements + * tv_grab_uk_tvguide: various fixes and improvements + * tv_grab_zz_sdjson_sqlite: + many fixes and improvements + 0.6.1 2019-02-21 * IMPORTANT * diff -Nru xmltv-0.6.1/debian/changelog xmltv-0.6.3/debian/changelog --- xmltv-0.6.1/debian/changelog 2020-05-23 20:42:50.000000000 +0000 +++ xmltv-0.6.3/debian/changelog 2020-11-08 11:07:44.000000000 +0000 @@ -1,8 +1,44 @@ -xmltv (0.6.1-1~ppa16.04+1) xenial; urgency=medium +xmltv (0.6.3-1~ppa16.04+1) xenial; urgency=medium - * Backppot from Debian Unstable. + * Backport from Debian Unstable. - -- Nicolas Derive Sat, 23 May 2020 22:42:50 +0200 + -- Nicolas Derive Sun, 08 Nov 2020 12:07:44 +0100 + +xmltv (0.6.3-1) unstable; urgency=medium + + * New upstream version 0.6.3 (Closes: #943932) + - tv_grab_dtv_la: grabber removed + - tv_grab_eu_dotmedia: grabber removed + - tv_grab_il: grabber removed + - tv_grab_pt_meo: grabber removed + - tv_grab_se_swedb: grabber removed + - tv_grab_se_tvzon: grabber removed + * d/control: + - declare compliance with Debian Policy 4.5.0 + - bump debhelper-compat level to 13 + - refresh (build) dependencies + - add Rules-Requires-Root field + - use my Debian email address for Maintainer + - update Vcs-* links + * d/copyright: + - Update years of Debian copyright + * d/doc-base: + - register tv_check docs with doc-base + * d/docs: + - update for filename change: README -> README.md + * d/install: + - refresh binary package lists after grabber changes + * d/lintian-overrides: + - drop xmltv-gui doc-base override + * d/NEWS: + - list alternatives for removed grabbers + * d/patches: + - refresh patches (upstream changes) + - update autopkgtest.patch: replace ADTTMP with AUTOPKGTEST_TMP + - add patch: ch_search_reenable + - add patch: typo-in-manual-page + + -- Nick Morrott Thu, 10 Sep 2020 00:38:01 +0100 xmltv (0.6.1-1) unstable; urgency=medium diff -Nru xmltv-0.6.1/debian/control xmltv-0.6.3/debian/control --- xmltv-0.6.1/debian/control 2020-05-23 20:42:39.000000000 +0000 +++ xmltv-0.6.3/debian/control 2020-11-08 11:07:38.000000000 +0000 @@ -1,7 +1,7 @@ Source: xmltv Section: interpreters Priority: optional -Maintainer: Nick Morrott +Maintainer: Nick Morrott Build-Depends: debhelper (>= 9) Build-Depends-Indep: libarchive-zip-perl, @@ -41,6 +41,7 @@ libtry-tiny-perl, libunicode-string-perl, liburi-perl, + liburi-encode-perl, libwww-perl, libxml-dom-perl, libxml-libxml-perl, @@ -52,11 +53,12 @@ libxml-writer-perl, perl, perl-tk, -Standards-Version: 4.3.0 +Standards-Version: 4.5.0 Testsuite: autopkgtest-pkg-perl -Vcs-Browser: https://salsa.debian.org/nickm-guest/xmltv -Vcs-Git: https://salsa.debian.org/nickm-guest/xmltv.git +Vcs-Browser: https://salsa.debian.org/nickm/xmltv +Vcs-Git: https://salsa.debian.org/nickm/xmltv.git Homepage: http://xmltv.org/ +Rules-Requires-Root: no Package: xmltv Architecture: all @@ -135,6 +137,7 @@ libtimedate-perl, libtry-tiny-perl, liburi-perl, + liburi-encode-perl, libwww-perl, libxml-dom-perl, libxml-libxml-perl, @@ -150,6 +153,7 @@ liblingua-preferred-perl, libterm-progressbar-perl, libunicode-string-perl, + liburi-escape-xs-perl, Suggests: libcgi-pm-perl, liblinux-dvb-perl, @@ -180,6 +184,8 @@ xmltv-util (= ${source:Version}), ${misc:Depends}, ${perl:Depends}, +Suggests: + doc-base, Description: graphical user interface for XMLTV tv_check utility Gather television listings, process them and organize your viewing. XMLTV is a file format for storing TV listings, defined in xmltv.dtd. diff -Nru xmltv-0.6.1/debian/copyright xmltv-0.6.3/debian/copyright --- xmltv-0.6.1/debian/copyright 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/copyright 2020-09-09 23:38:01.000000000 +0000 @@ -67,7 +67,7 @@ Files: debian/* Copyright: 2002-2006 Kenneth Pronovici 2006-2012 Chris Butler - 2015-2019 Nick Morrott + 2015-2020 Nick Morrott License: GPL-2 License: GPL-2 diff -Nru xmltv-0.6.1/debian/libxmltv-perl.docs xmltv-0.6.3/debian/libxmltv-perl.docs --- xmltv-0.6.1/debian/libxmltv-perl.docs 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/libxmltv-perl.docs 2020-09-09 23:38:01.000000000 +0000 @@ -1,3 +1,3 @@ debian/README.Debian -README +README.md doc/QuickStart diff -Nru xmltv-0.6.1/debian/libxmltv-perl.install xmltv-0.6.3/debian/libxmltv-perl.install --- xmltv-0.6.1/debian/libxmltv-perl.install 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/libxmltv-perl.install 2020-09-09 23:38:01.000000000 +0000 @@ -1,21 +1,21 @@ -choose/tv_pick/tv_pick_cgi usr/share/doc/libxmltv-perl/examples -debian/tmp/usr/share/man/man3/XMLTV.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Augment.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Configure.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Configure::Writer.3pm usr/share/man/man3 +choose/tv_pick/tv_pick_cgi usr/share/doc/libxmltv-perl/examples +debian/tmp/usr/share/man/man3/XMLTV.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Augment.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Configure.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Configure::Writer.3pm usr/share/man/man3 debian/tmp/usr/share/man/man3/XMLTV::Data::Recursive::Encode.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Date.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::GUI.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Grab_XML.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Gunzip.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Options.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::PreferredMethod.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Summarize.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Supplement.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::ValidateFile.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::ValidateGrabber.3pm usr/share/man/man3 -debian/tmp/usr/share/man/man3/XMLTV::Version.3pm usr/share/man/man3 -debian/tmp/usr/share/perl5/XMLTV usr/share/perl5 -debian/tmp/usr/share/perl5/XMLTV.pm usr/share/perl5 -xmltv.dtd usr/share/sgml/xmltv/dtd/0.5 -xmltv-lineups.xsd usr/share/sgml/xmltv/dtd/0.5 +debian/tmp/usr/share/man/man3/XMLTV::Date.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::GUI.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Grab_XML.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Gunzip.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Options.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::PreferredMethod.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Summarize.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Supplement.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::ValidateFile.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::ValidateGrabber.3pm usr/share/man/man3 +debian/tmp/usr/share/man/man3/XMLTV::Version.3pm usr/share/man/man3 +debian/tmp/usr/share/perl5/XMLTV usr/share/perl5 +debian/tmp/usr/share/perl5/XMLTV.pm usr/share/perl5 +xmltv.dtd usr/share/sgml/xmltv/dtd/0.5 +xmltv-lineups.xsd usr/share/sgml/xmltv/dtd/0.5 diff -Nru xmltv-0.6.1/debian/NEWS xmltv-0.6.3/debian/NEWS --- xmltv-0.6.1/debian/NEWS 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/NEWS 2020-09-09 23:38:01.000000000 +0000 @@ -1,22 +1,12 @@ -xmltv (0.6.1-1) unstable; urgency=medium +xmltv (0.6.3-1) unstable; urgency=medium - Important notice for users of the tv_grab_eu_dotmedia or - tv_grab_se_tvzon grabbers: + The following grabbers were removed in this release: - tv_grab_eu_dotmedia and tv_grab_se_tvzon are now deprecated and will - be removed in the next release of XMLTV. Please switch to the new - tv_grab_eu_xmltvse grabber as soon as possible. + - tv_grab_dtv_la + - tv_grab_eu_dotmedia - please use tv_grab_eu_xmltvse + - tv_grab_il + - tv_grab_pt_meo - please use tv_grab_pt_vodafone + - tv_grab_se_swedb - please use tv_grab_eu_xmltvse + - tv_grab_se_tvzon - please use tv_grab_eu_xmltvse - -- Nick Morrott Thu, 28 Feb 2019 23:12:00 +0000 - -xmltv (0.5.70-1) unstable; urgency=medium - - Important notice for users of the tv_grab_sd_json grabber: - - This and future releases of XMLTV no longer provide the tv_grab_sd_json - grabber, which has been renamed tv_grab_zz_sdjson. Existing users of the - tv_grab_sd_json grabber must migrate to the tv_grab_zz_sdjson grabber - immediately in order to continue to receive listings. Existing config - files should continue to work. - - -- Nick Morrott Thu, 30 Nov 2017 16:29:17 +0000 + -- Nick Morrott Mon, 07 Sep 2020 16:22:30 +0100 diff -Nru xmltv-0.6.1/debian/patches/autopkgtest.patch xmltv-0.6.3/debian/patches/autopkgtest.patch --- xmltv-0.6.1/debian/patches/autopkgtest.patch 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/patches/autopkgtest.patch 2020-09-09 23:38:01.000000000 +0000 @@ -1,7 +1,7 @@ Description: Patch source tests to make them autopkgtest-able -Author: Nick Morrott +Author: Nick Morrott Forwarded: not-needed -Last-Update: 2019-03-01 +Last-Update: 2020-09-07 --- --- a/t/test_filters.t +++ b/t/test_filters.t @@ -10,18 +10,18 @@ my $tests_dir = 't/data'; # directory test files live in die "no directory $tests_dir" if not -d $tests_dir; -my $cmds_dir = 'blib/script'; # directory filter programs live in -+my $cmds_dir = $ENV{ADTTMP} ? '/usr/bin' : 'blib/script'; # directory filter programs live in ++my $cmds_dir = $ENV{AUTOPKGTEST_TMP} ? '/usr/bin' : 'blib/script'; # directory filter programs live in die "no directory $cmds_dir" if not -d $cmds_dir; my $verbose = 0; --- a/t/test_tv_imdb.t +++ b/t/test_tv_imdb.t -@@ -21,7 +21,7 @@ +@@ -18,7 +18,7 @@ my $tests_dir = 't/data-tv_imdb'; # where to find input XML files die "no directory $tests_dir" if not -d $tests_dir; --my $cmds_dir = 'blib/script'; # directory tv_split lives in -+my $cmds_dir = $ENV{ADTTMP} ? '/usr/bin' : 'blib/script'; # directory tv_split lives in +-my $cmds_dir = 'blib/script'; # directory tv_imdb lives in ++my $cmds_dir = $ENV{AUTOPKGTEST_TMP} ? '/usr/bin' : 'blib/script'; # directory tv_imdb lives in die "no directory $cmds_dir" if not -d $cmds_dir; my $verbose = 0; @@ -32,7 +32,7 @@ my $tests_dir = 't/data'; # where to find input XML files die "no directory $tests_dir" if not -d $tests_dir; -my $cmds_dir = 'blib/script'; # directory tv_split lives in -+my $cmds_dir = $ENV{ADTTMP} ? '/usr/bin' : 'blib/script'; # directory tv_split lives in ++my $cmds_dir = $ENV{AUTOPKGTEST_TMP} ? '/usr/bin' : 'blib/script'; # directory tv_split lives in die "no directory $cmds_dir" if not -d $cmds_dir; my $verbose = 0; @@ -43,7 +43,7 @@ my $tests_dir = 't/data-tv_augment'; # where to find input XML files die "no directory $tests_dir" if not -d $tests_dir; -my $cmds_dir = 'blib/script'; # directory tv_augment lives in -+my $cmds_dir = $ENV{ADTTMP} ? '/usr/bin' : 'blib/script'; # directory filter programs live in ++my $cmds_dir = $ENV{AUTOPKGTEST_TMP} ? '/usr/bin' : 'blib/script'; # directory filter programs live in die "no directory $cmds_dir" if not -d $cmds_dir; my $verbose = 0; diff -Nru xmltv-0.6.1/debian/patches/ch_search_reenable xmltv-0.6.3/debian/patches/ch_search_reenable --- xmltv-0.6.1/debian/patches/ch_search_reenable 1970-01-01 00:00:00.000000000 +0000 +++ xmltv-0.6.3/debian/patches/ch_search_reenable 2020-09-09 23:38:01.000000000 +0000 @@ -0,0 +1,53 @@ +Description: Re-enable tv_grab_ch_search after upstream patch +Author: Patric Mueller +Origin: upstream +Bug: https://github.com/XMLTV/xmltv/issues/109 +Applied-Upstream: cb029fc6ea0b7ec688d7b881806e699119353458 +Last-Update: 2020-09-07 +--- +--- a/Makefile.PL ++++ b/Makefile.PL +@@ -280,19 +280,19 @@ + 'HTTP::Cookies' => 0, }, + }, + +- # { name => 'tv_grab_ch_search', +- # blurb => 'Grabber for Switzerland', +- # exes => [ 'grab/ch_search/tv_grab_ch_search' ], +- # deps => [ 'grab/ch_search/tv_grab_ch_search' => [ 'grab/ch_search/tv_grab_ch_search.in' ] ], +- # pl_files => { 'grab/ch_search/tv_grab_ch_search.PL' => 'grab/ch_search/tv_grab_ch_search' }, +- # to_clean => [ 'grab/ch_search/tv_grab_ch_search' ], +- # grab_need_share => [ 'ch_search' ], +- # prereqs => { 'HTML::Entities' => 1.27, +- # 'HTML::TreeBuilder' => 0, +- # 'HTTP::Cookies' => 0, +- # 'URI::Escape' => 0, +- # 'URI::URL' => 0, }, +- # }, ++ { name => 'tv_grab_ch_search', ++ blurb => 'Grabber for Switzerland', ++ exes => [ 'grab/ch_search/tv_grab_ch_search' ], ++ deps => [ 'grab/ch_search/tv_grab_ch_search' => [ 'grab/ch_search/tv_grab_ch_search.in' ] ], ++ pl_files => { 'grab/ch_search/tv_grab_ch_search.PL' => 'grab/ch_search/tv_grab_ch_search' }, ++ to_clean => [ 'grab/ch_search/tv_grab_ch_search' ], ++ grab_need_share => [ 'ch_search' ], ++ prereqs => { 'HTML::Entities' => 1.27, ++ 'HTML::TreeBuilder' => 0, ++ 'HTTP::Cookies' => 0, ++ 'URI::Escape' => 0, ++ 'URI::URL' => 0, }, ++ }, + + { name => 'tv_grab_dk_dr', + blurb => 'Grabber for Denmark (dr.dk)', +--- a/grab/ch_search/tv_grab_ch_search.in ++++ b/grab/ch_search/tv_grab_ch_search.in +@@ -374,7 +374,7 @@ + foreach my $tv_channel ( $tb->look_down('class' => 'sl-card tv-index-channel') ) { + my $channel_id = substr($tv_channel->attr('id'), 3); # tv-sf1 -> sf1 + if ( defined($channel_id) ) { +- foreach my $tv_show ( $tv_channel ->look_down('class' => 'tv-tooltip') ) { ++ foreach my $tv_show ( $tv_channel ->look_down('class', qr/(^| )tv-tooltip( |$)/) ) { + my %show; + $show{channel} = channel_id($channel_id); + diff -Nru xmltv-0.6.1/debian/patches/manpage-has-errors-from-pod2man xmltv-0.6.3/debian/patches/manpage-has-errors-from-pod2man --- xmltv-0.6.1/debian/patches/manpage-has-errors-from-pod2man 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/patches/manpage-has-errors-from-pod2man 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ -Description: Fix POD warnings -Author: Nick Morrott -Applied-Upstream: https://github.com/XMLTV/xmltv/commit/5162eeacc2f34e007690a2d58d8aece9c80ca58a -Last-Update: 2019-03-01 ---- ---- a/grab/pt_vodafone/tv_grab_pt_vodafone -+++ b/grab/pt_vodafone/tv_grab_pt_vodafone -@@ -2,6 +2,8 @@ - - =pod - -+=encoding utf8 -+ - =head1 NAME - - tv_grab_pt_vodafone - Grab TV listings for Vodafone in Portugal. -@@ -76,17 +78,21 @@ - it might be necessary to set HOME to a path without spaces in it. - - =head1 CREDITS -+ - Kevin Groeneveld (kgroeneveld at gmail dot com) - --This grabber uses code from tv_grab_pt_meo by Karl Dietz, dekarl -at- users -dot- sourceforge -dot- net. --This grabber uses code from tv_grab_zz_sdjson by Kevin Groeneveld, kgroeneveld -at- gmail -dot- com. --The original idea of this grabber came from higuita's shell script, --see L. --Special thanks to Vodafone, for building a clean, fast and public access API, --much more reliable than Meo open API ( but sadly not as open) and much better --than lack of any API from NOS. -+This grabber uses code from tv_grab_pt_meo by Karl Dietz, dekarl -at- users -dot- sourceforge -dot- net, -+and from tv_grab_zz_sdjson by Kevin Groeneveld, kgroeneveld -at- gmail -dot- com. -+ -+The original idea of this grabber came from higuita's shell script, see -+L. -+ -+Special thanks to Vodafone for building a clean, fast, and public access API; -+much more reliable than Meo's open API (but sadly not as open) and much better -+than the lack of any API from NOS. - - =head1 AUTHOR -+ - Nuno Sénica, nsenica -at- gmail -dot- com. - - =head1 BUGS diff -Nru xmltv-0.6.1/debian/patches/series xmltv-0.6.3/debian/patches/series --- xmltv-0.6.1/debian/patches/series 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/patches/series 2020-09-09 23:38:01.000000000 +0000 @@ -1,6 +1,5 @@ it_dvb_linux_warning 11_makefile_pl_debian_changes.diff autopkgtest.patch -spelling-error-in-manpage.patch -test_tv_imdb_no_redirection -manpage-has-errors-from-pod2man +ch_search_reenable +typo-in-manual-page diff -Nru xmltv-0.6.1/debian/patches/spelling-error-in-manpage.patch xmltv-0.6.3/debian/patches/spelling-error-in-manpage.patch --- xmltv-0.6.1/debian/patches/spelling-error-in-manpage.patch 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/patches/spelling-error-in-manpage.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -Description: Fix spelling error detected by lintian -Author: Nick Morrott -Applied-Upstream: https://github.com/XMLTV/xmltv/commit/9dc7ae7d52e9d44935fb49f980d9f1280594d8b1 -Last-Update: 2019-03-01 ---- ---- a/filter/tv_imdb -+++ b/filter/tv_imdb -@@ -151,7 +151,7 @@ - Needs some more controls for fine tuning "close" matches. For - instance, currently it looks like the North America grabber only has - date entries for movies, but the imdb.com data contains made for video --movies as well as as real movies, ot is itE<39>s possible to get the -+movies as well as as real movies, so itE<39>s possible to get the - wrong data to be inserted. In this case we may want to say "ignore tv - series" and "ignore tv mini series". Along with this, weE<39>d want - to define what a "close" match is. For instance does a movie by the diff -Nru xmltv-0.6.1/debian/patches/test_tv_imdb_no_redirection xmltv-0.6.3/debian/patches/test_tv_imdb_no_redirection --- xmltv-0.6.1/debian/patches/test_tv_imdb_no_redirection 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/patches/test_tv_imdb_no_redirection 1970-01-01 00:00:00.000000000 +0000 @@ -1,25 +0,0 @@ -Description: Do not use STDIN/STDOUT redirection for input/output file - Use the documented --output option for output filename, rather than relying - on the redirection of input/ouput. - . - This issue was revealed after changes to libxml-parser-perl (see bug report - below) to address its use of 2-argument open(). -Author: Nick Morrott -Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=923223 -Applied-Upstream: https://github.com/XMLTV/xmltv/commit/5463cde27030237d79fedd200e49968edaa06f67 -Last-Update: 2019-03-01 ---- ---- a/t/test_tv_imdb.t -+++ b/t/test_tv_imdb.t -@@ -84,9 +84,9 @@ - my $output="$tmpDir/".File::Basename::basename($input)."-output.xml"; - - # Make temporary directory and split into it. -- my $cmd="$cmds_dir/tv_imdb --quiet --imdbdir '$tmpDir' --with-keywords --with-plot < $input > '$output' 2>&1"; -+ my $cmd="$cmds_dir/tv_imdb --quiet --imdbdir '$tmpDir' --with-keywords --with-plot --output '$output' '$input' 2>&1"; - if ( $input=~m/movies-only/ ) { -- $cmd="$cmds_dir/tv_imdb --movies-only --quiet --imdbdir '$tmpDir' --with-keywords --with-plot < $input > '$output' 2>&1"; -+ $cmd="$cmds_dir/tv_imdb --movies-only --quiet --imdbdir '$tmpDir' --with-keywords --with-plot --output '$output' '$input' 2>&1"; - } - #print STDERR "\nRUN:$cmd\n"; - my $r = system($cmd); diff -Nru xmltv-0.6.1/debian/patches/typo-in-manual-page xmltv-0.6.3/debian/patches/typo-in-manual-page --- xmltv-0.6.1/debian/patches/typo-in-manual-page 1970-01-01 00:00:00.000000000 +0000 +++ xmltv-0.6.3/debian/patches/typo-in-manual-page 2020-09-09 23:38:01.000000000 +0000 @@ -0,0 +1,40 @@ +Description: Fix some typos detected by lintian +Author: Nick Morrott +Forwarded: not-needed +Applied-Upstream: ea01fcb293b1d95ce89fe055bef38b54253de26e +Last-Update: 2020-09-09 +--- +--- a/filter/tv_imdb ++++ b/filter/tv_imdb +@@ -32,9 +32,9 @@ + + B<--output FILE> write to FILE rather than standard output. + +-B<--with-keywords> include IDMb keywords in the output file. ++B<--with-keywords> include IMDb keywords in the output file. + +-B<--with-plot> include IDMb plot summary in the output file. ++B<--with-plot> include IMDb plot summary in the output file. + + B<--actors NUMBER> number of actors from IMDb to add (default=3). + +@@ -115,7 +115,7 @@ + hand. See for the download sites. + Then once you have the files rerun without '--download'. + +-Note: '--prepStage' sucks a bit of memeory, but you can run each ++Note: '--prepStage' sucks a bit of memory, but you can run each + prepStage separately by running --prepStage with each of the stages + (see --help for details). + +--- a/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite ++++ b/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite +@@ -6129,7 +6129,7 @@ + + B<--force-download> Deletes most existing local database data and + forces a download of the data. If there is a suspicion that the +-data is currupt (and not being automatically corrected), forcing ++data is corrupt (and not being automatically corrected), forcing + a new download might be necessary. + + B<--days N> When grabbing, grab N days rather than all available days. diff -Nru xmltv-0.6.1/debian/xmltv.docs xmltv-0.6.3/debian/xmltv.docs --- xmltv-0.6.1/debian/xmltv.docs 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv.docs 2020-09-09 23:38:01.000000000 +0000 @@ -1,3 +1,3 @@ debian/README.Debian -README +README.md doc/QuickStart diff -Nru xmltv-0.6.1/debian/xmltv-gui.doc-base xmltv-0.6.3/debian/xmltv-gui.doc-base --- xmltv-0.6.1/debian/xmltv-gui.doc-base 1970-01-01 00:00:00.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv-gui.doc-base 2020-09-09 23:38:01.000000000 +0000 @@ -0,0 +1,11 @@ +Document: xmltv-gui +Title: TV-CHECK User Guide +Author: Robert Eden +Abstract: TV-CHECK compares a listing of your favorite TV shows against an + actual XMLTV broadcast schedule, and reports changes in your favorite + shows as well as any extra episodes. +Section: TV and Radio + +Format: HTML +Index: /usr/share/doc/xmltv-gui/tv_check_doc.html +Files: /usr/share/doc/xmltv-gui/*.html diff -Nru xmltv-0.6.1/debian/xmltv-gui.docs xmltv-0.6.3/debian/xmltv-gui.docs --- xmltv-0.6.1/debian/xmltv-gui.docs 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv-gui.docs 2020-09-09 23:38:01.000000000 +0000 @@ -1,5 +1,5 @@ debian/README.Debian -README +README.md doc/QuickStart choose/tv_check/README.tv_check choose/tv_check/tv_check_doc.html diff -Nru xmltv-0.6.1/debian/xmltv-gui.lintian-overrides xmltv-0.6.3/debian/xmltv-gui.lintian-overrides --- xmltv-0.6.1/debian/xmltv-gui.lintian-overrides 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv-gui.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -# A single html page and graphic is installed to /usr/share/doc -xmltv-gui: possible-documentation-but-no-doc-base-registration diff -Nru xmltv-0.6.1/debian/xmltv-util.docs xmltv-0.6.3/debian/xmltv-util.docs --- xmltv-0.6.1/debian/xmltv-util.docs 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv-util.docs 2020-09-09 23:38:01.000000000 +0000 @@ -1,3 +1,3 @@ debian/README.Debian -README +README.md doc/QuickStart diff -Nru xmltv-0.6.1/debian/xmltv-util.install xmltv-0.6.3/debian/xmltv-util.install --- xmltv-0.6.1/debian/xmltv-util.install 2019-03-01 01:52:30.000000000 +0000 +++ xmltv-0.6.3/debian/xmltv-util.install 2020-09-09 23:38:01.000000000 +0000 @@ -9,25 +9,19 @@ debian/tmp/usr/bin/tv_grab_ch_search usr/bin debian/tmp/usr/bin/tv_grab_combiner usr/bin debian/tmp/usr/bin/tv_grab_dk_dr usr/bin -debian/tmp/usr/bin/tv_grab_dtv_la usr/bin -debian/tmp/usr/bin/tv_grab_eu_dotmedia usr/bin debian/tmp/usr/bin/tv_grab_eu_epgdata usr/bin debian/tmp/usr/bin/tv_grab_eu_xmltvse usr/bin debian/tmp/usr/bin/tv_grab_fi usr/bin debian/tmp/usr/bin/tv_grab_fi_sv usr/bin debian/tmp/usr/bin/tv_grab_fr usr/bin debian/tmp/usr/bin/tv_grab_huro usr/bin -debian/tmp/usr/bin/tv_grab_il usr/bin debian/tmp/usr/bin/tv_grab_is usr/bin debian/tmp/usr/bin/tv_grab_it usr/bin debian/tmp/usr/bin/tv_grab_it_dvb usr/bin debian/tmp/usr/bin/tv_grab_na_dd usr/bin debian/tmp/usr/bin/tv_grab_na_dtv usr/bin debian/tmp/usr/bin/tv_grab_na_tvmedia usr/bin -debian/tmp/usr/bin/tv_grab_pt_meo usr/bin debian/tmp/usr/bin/tv_grab_pt_vodafone usr/bin -debian/tmp/usr/bin/tv_grab_se_swedb usr/bin -debian/tmp/usr/bin/tv_grab_se_tvzon usr/bin debian/tmp/usr/bin/tv_grab_tr usr/bin debian/tmp/usr/bin/tv_grab_uk_bleb usr/bin debian/tmp/usr/bin/tv_grab_uk_tvguide usr/bin @@ -55,25 +49,19 @@ debian/tmp/usr/share/man/man1/tv_grab_ch_search.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_combiner.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_dk_dr.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_dtv_la.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_eu_dotmedia.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_eu_epgdata.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_eu_xmltvse.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_fi.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_fi_sv.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_fr.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_huro.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_il.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_is.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_it.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_it_dvb.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_na_dd.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_na_dtv.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_na_tvmedia.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_pt_meo.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_pt_vodafone.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_se_swedb.1p usr/share/man/man1 -debian/tmp/usr/share/man/man1/tv_grab_se_tvzon.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_tr.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_uk_bleb.1p usr/share/man/man1 debian/tmp/usr/share/man/man1/tv_grab_uk_tvguide.1p usr/share/man/man1 diff -Nru xmltv-0.6.1/doc/QuickStart xmltv-0.6.3/doc/QuickStart --- xmltv-0.6.1/doc/QuickStart 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/doc/QuickStart 2020-09-07 15:02:53.000000000 +0000 @@ -13,19 +13,15 @@ Czech Republic tv_grab_cz Denmark tv_grab_dk_dr, tv_grab_dk_tvtid Finland tv_grab_fi, tv_grab_fi_sv - France tv_grab_fr, tv_grab_fr_kazer + France tv_grab_fr Germany tv_grab_de Hungary and Romania tv_grab_huro Iceland tv_grab_is - India tv_grab_in_toi Israel tv_grab_il Italy tv_grab_it, tv_grab_it_dvb Japan tv_grab_jp - Netherlands tv_grab_nl Norway tv_grab_no - Portugal tv_grab_pt_meo, tv_grab_pt_vodafone - Spain tv_grab_es_laguiatv - Sweden tv_grab_se_swedb, tv_grab_se_tvzon + Portugal tv_grab_pt_vodafone Switzerland tv_grab_ch_search Turkey tv_grab_tr UK and Ireland tv_grab_uk_bleb, tv_grab_uk_tvguide @@ -35,8 +31,7 @@ Europe/US/Canda/Latin America/Caribbean tv_grab_zz_sdjson, tv_grab_zz_sdjson_sqlite - Europe tv_grab_eu_dotmedia, tv_grab_eu_epgdata - Latin America tv_grab_dtv_la + Europe tv_grab_eu_epgdata Contributions from other countries are welcome, of course. diff -Nru xmltv-0.6.1/doc/README.win32 xmltv-0.6.3/doc/README.win32 --- xmltv-0.6.1/doc/README.win32 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/doc/README.win32 2020-09-07 15:02:53.000000000 +0000 @@ -1,4 +1,4 @@ -XMLTV 0.6.1, Windows binary release +XMLTV 0.6.3, Windows binary release Gather television listings, process them and organize your viewing. XMLTV is a file format for storing TV listings, defined in xmltv.dtd. @@ -14,46 +14,40 @@ * Known problems Due to prerequisite problems, EXE support is not currently available -for tv_grab_is, tv_grab_pt_meo and tv_grab_it_dvb, If you need one +for tv_grab_is and tv_grab_it_dvb, If you need one of those you'll need to install Perl and the necessary modules and use the full distribution. -Some Win98 users get errors in Perl58.dll when running tv_grab_na_dd. See -below for more info. +* Major Changes in this release (0.6.3) -tv_grab_se_swedb sometimes fails to work on Windows if there are spaces -in the path to your home-directory. This can be avoided by setting -the environment variable HOME to a path without spaces (e.g. c:\home). +tv_grab_ch_search: disable broken grabber -* Major Changes in this release (0.6.1) +* Major Changes in previous release (0.6.2) -* IMPORTANT * +tv_grab_dotmedia: disable deprecated grabber +tv_grab_se_tvzon: disable deprecated grabber -tv_grab_eu_dotmedia and tv_grab_se_tvzon are deprecated and will be -removed in the next release of XMLTV. Please switch to the new -tv_grab_eu_xmltvse grabber as soon as possible. - -tv_grab_eu_xmltvse: new grabber for Europe -tv_grab_pt_vodafone: new grabber for Portugal - -tv_grab_es_laguiatv: disable broken grabber -tv_grab_fr_kazer: disable broken grabber -tv_grab_in_toi: disable broken grabber -tv_grab_nl: disable broken grabber - -tv_grab_eu_epgdata: include fanart URLs in output -tv_grab_fi: add new ampparit and telsu sources -tv_grab_il: update grabber due to upstream changes -tv_grab_is: now only provides RUV channels +tv_grab_dtv_la: disable broken grabber +tv_grab_il: disable broken grabber +tv_grab_pt_meo: disable broken grabber +tv_grab_se_swedb: disable broken grabber + +XMLTV.pm: update handling of reading from STDIN due to + XML::Parser adopting 3-arg open +tv_grab_ch_search: handle upstream cookies +tv_grab_eu_epgdata: various fixes and improvements +tv_grab_fi: various fixes and improvements +tv_grab_fr: update grabber due to upstream changes +tv_grab_huro: use https source site URLs +tv_grab_it: fix overlapping/duplicate programmes +tv_grab_na_dd: use https source site URLs +tv_grab_na_dtv: various fixes and improvements +tv_grab_pt_vodafone: various fixes and improvements +tv_grab_uk_tvguide: various fixes and improvements tv_grab_zz_sdjson_sqlite: - improvements to lineup management - add support for TheTVDB metadata -tv_augment: new rules to improve episode numbering - logging must now be enabled explicitly -tv_count/tv_merge: mandatory command line options for files -tv_imdb: migrate to new URL for archived IMDB data + many fixes and improvements -And lots of other changes (see git log for details) +And many other changes (see the git log for details) There is no installer, just unpack the zipfile into a directory such as C:\xmltv. If you are reading this you've probably @@ -91,29 +85,6 @@ If you *DO* want to insert a shell variable, you can do so by surrounding it with percents. (ex %HOME% ) -* crash in with PERL58.DLL - -When using tv_grab_na_dd, some users are experiencing a crash in the -Perl58.DLL. - -I had a similar problem when I had an older version of Perl installed -as well as XMLTV. It seems to cause DLL confusion and eventually a program -crash. - -I fixed my system when I removed the old Perl. Re-installing Perl -didn't cause the problem to return. - -Before that, I used the work-around below (I could fetch 3 days) - -tv_grab_na_dd --output=a.xml --days 3 -tv_grab_na_dd --output=b.xml --days 3 --offset 3 -tv_grab_na_dd --output=c.xml --days 1 --offset 6 -tv_cat a.xml b.xml c.xml >guide.xml - -I really thought this problem would be a fluke and/or easily solved -when I solved it for myself. That doesn't look to be the case. I'm -looking for a solution. - * Proxy servers Proxy server support is provide by the LWP modules. @@ -140,4 +111,4 @@ xmltv-users - how to use XMLTV xmltv-devel - detailed discussions among developers --- Nick Morrott, knowledgejunkie@gmail.com, 2019-02-21 +-- Nick Morrott, knowledgejunkie@gmail.com, 2020-08-22 diff -Nru xmltv-0.6.1/.Dockerfile.centos-6 xmltv-0.6.3/.Dockerfile.centos-6 --- xmltv-0.6.1/.Dockerfile.centos-6 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.centos-6 1970-01-01 00:00:00.000000000 +0000 @@ -1,58 +0,0 @@ -# Dockerfile to generate CentOS 6 build container for XMLTV - -FROM centos:6 - -MAINTAINER Nick Morrott - -RUN yum -y install epel-release - -RUN yum -y install \ - make \ - perl-core \ - perl-Archive-Zip \ - perl-Crypt-SSLeay \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Temp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.centos-7 xmltv-0.6.3/.Dockerfile.centos-7 --- xmltv-0.6.1/.Dockerfile.centos-7 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.centos-7 1970-01-01 00:00:00.000000000 +0000 @@ -1,57 +0,0 @@ -# Dockerfile to generate CentOS 7 build container for XMLTV - -FROM centos:7 - -MAINTAINER Nick Morrott - -RUN yum -y install epel-release - -RUN yum -y install \ - make \ - perl-core \ - perl-Archive-Zip \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.debian-buster xmltv-0.6.3/.Dockerfile.debian-buster --- xmltv-0.6.1/.Dockerfile.debian-buster 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.debian-buster 1970-01-01 00:00:00.000000000 +0000 @@ -1,61 +0,0 @@ -# Dockerfile to generate Debian 10 "Buster" build container for XMLTV - -FROM debian:buster - -MAINTAINER Nick Morrott - -RUN echo "deb http://deb.debian.org/debian buster main" > /etc/apt/sources.list - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - --no-install-recommends diff -Nru xmltv-0.6.1/.Dockerfile.debian-jessie xmltv-0.6.3/.Dockerfile.debian-jessie --- xmltv-0.6.1/.Dockerfile.debian-jessie 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.debian-jessie 1970-01-01 00:00:00.000000000 +0000 @@ -1,62 +0,0 @@ -# Dockerfile to generate Debian 8 "Jessie" build container for XMLTV - -FROM debian:jessie - -MAINTAINER Nick Morrott - -RUN echo "deb http://deb.debian.org/debian jessie main" > /etc/apt/sources.list -RUN echo "deb http://deb.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - --no-install-recommends diff -Nru xmltv-0.6.1/.Dockerfile.debian-sid xmltv-0.6.3/.Dockerfile.debian-sid --- xmltv-0.6.1/.Dockerfile.debian-sid 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.debian-sid 1970-01-01 00:00:00.000000000 +0000 @@ -1,61 +0,0 @@ -# Dockerfile to generate Debian Sid build container for XMLTV - -FROM debian:sid - -MAINTAINER Nick Morrott - -RUN echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - --no-install-recommends diff -Nru xmltv-0.6.1/.Dockerfile.debian-stretch xmltv-0.6.3/.Dockerfile.debian-stretch --- xmltv-0.6.1/.Dockerfile.debian-stretch 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.debian-stretch 1970-01-01 00:00:00.000000000 +0000 @@ -1,62 +0,0 @@ -# Dockerfile to generate Debian 9 "Stretch" build container for XMLTV - -FROM debian:stretch - -MAINTAINER Nick Morrott - -RUN echo "deb http://deb.debian.org/debian stretch main" > /etc/apt/sources.list -RUN echo "deb http://deb.debian.org/debian-security stretch/updates main" >> /etc/apt/sources.list - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - --no-install-recommends diff -Nru xmltv-0.6.1/.Dockerfile.fedora-24 xmltv-0.6.3/.Dockerfile.fedora-24 --- xmltv-0.6.1/.Dockerfile.fedora-24 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-24 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 24 build container for XMLTV - -FROM fedora:24 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl-core \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.fedora-25 xmltv-0.6.3/.Dockerfile.fedora-25 --- xmltv-0.6.1/.Dockerfile.fedora-25 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-25 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 25 build container for XMLTV - -FROM fedora:25 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl-core \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.fedora-26 xmltv-0.6.3/.Dockerfile.fedora-26 --- xmltv-0.6.1/.Dockerfile.fedora-26 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-26 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 26 build container for XMLTV - -FROM fedora:26 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl-core \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.fedora-27 xmltv-0.6.3/.Dockerfile.fedora-27 --- xmltv-0.6.1/.Dockerfile.fedora-27 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-27 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 27 build container for XMLTV - -FROM fedora:27 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.fedora-28 xmltv-0.6.3/.Dockerfile.fedora-28 --- xmltv-0.6.1/.Dockerfile.fedora-28 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-28 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 28 build container for XMLTV - -FROM fedora:28 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.fedora-29 xmltv-0.6.3/.Dockerfile.fedora-29 --- xmltv-0.6.1/.Dockerfile.fedora-29 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.fedora-29 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -# Dockerfile to generate Fedora 29 build container for XMLTV - -FROM fedora:29 - -MAINTAINER Nick Morrott - -RUN yum -y install \ - make \ - perl \ - perl-Archive-Zip \ - perl-CGI \ - perl-Data-Dump \ - perl-Date-Calc \ - perl-Date-Manip \ - perl-DateTime \ - perl-DateTime-Format-ISO8601 \ - perl-DateTime-Format-SQLite \ - perl-DateTime-Format-Strptime \ - perl-DBD-SQLite \ - perl-DBI \ - perl-File-chdir \ - perl-File-HomeDir \ - perl-File-Slurp \ - perl-File-Which \ - perl-HTML-Parser \ - perl-HTML-Tree \ - perl-HTTP-Cache-Transparent \ - perl-HTTP-Cookies \ - perl-HTTP-Message \ - perl-IO-stringy \ - perl-JSON \ - perl-JSON-XS \ - perl-libwww-perl \ - perl-Lingua-Preferred \ - perl-List-MoreUtils \ - perl-Log-TraceMessages \ - perl-LWP-Protocol-https \ - perl-LWP-UserAgent-Determined \ - perl-PerlIO-gzip \ - perl-SOAP-Lite \ - perl-Term-ProgressBar \ - perl-TermReadKey \ - perl-TimeDate \ - perl-Tk \ - perl-Tk-TableMatrix \ - perl-Try-Tiny \ - perl-Unicode-String \ - perl-URI \ - perl-XML-DOM \ - perl-XML-LibXML \ - perl-XML-LibXSLT \ - perl-XML-Parser \ - perl-XML-Simple \ - perl-XML-TreePP \ - perl-XML-Twig \ - perl-XML-Writer diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-artful xmltv-0.6.3/.Dockerfile.ubuntu-artful --- xmltv-0.6.1/.Dockerfile.ubuntu-artful 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-artful 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -# Dockerfile to generate Ubuntu 17.10 "Artful Aardvark" build container for XMLTV - -FROM ubuntu:artful - -MAINTAINER Nick Morrott - -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-bionic xmltv-0.6.3/.Dockerfile.ubuntu-bionic --- xmltv-0.6.1/.Dockerfile.ubuntu-bionic 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-bionic 1970-01-01 00:00:00.000000000 +0000 @@ -1,69 +0,0 @@ -# Dockerfile to generate Ubuntu 18.04 LTS "Bionic Beaver" build container for XMLTV - -FROM ubuntu:bionic - -MAINTAINER Nick Morrott - -ENV DEBIAN_FRONTEND 'noninteractive' -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-cosmic xmltv-0.6.3/.Dockerfile.ubuntu-cosmic --- xmltv-0.6.1/.Dockerfile.ubuntu-cosmic 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-cosmic 1970-01-01 00:00:00.000000000 +0000 @@ -1,69 +0,0 @@ -# Dockerfile to generate Ubuntu 18.10 "Cosmic Cuttlefish" build container for XMLTV - -FROM ubuntu:cosmic - -MAINTAINER Nick Morrott - -ENV DEBIAN_FRONTEND 'noninteractive' -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-trusty xmltv-0.6.3/.Dockerfile.ubuntu-trusty --- xmltv-0.6.1/.Dockerfile.ubuntu-trusty 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-trusty 1970-01-01 00:00:00.000000000 +0000 @@ -1,59 +0,0 @@ -# Dockerfile to generate Ubuntu 14.04 LTS "Trusty Tahr" build container for XMLTV - -FROM ubuntu:trusty - -MAINTAINER Nick Morrott - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - --no-install-recommends diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-xenial xmltv-0.6.3/.Dockerfile.ubuntu-xenial --- xmltv-0.6.1/.Dockerfile.ubuntu-xenial 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-xenial 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -# Dockerfile to generate Ubuntu 16.04 LTS "Xenial Xerus" build container for XMLTV - -FROM ubuntu:xenial - -MAINTAINER Nick Morrott - -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-yakkety xmltv-0.6.3/.Dockerfile.ubuntu-yakkety --- xmltv-0.6.1/.Dockerfile.ubuntu-yakkety 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-yakkety 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -# Dockerfile to generate Ubuntu 16.10 "Yakkety Yak" build container for XMLTV - -FROM ubuntu:yakkety - -MAINTAINER Nick Morrott - -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.Dockerfile.ubuntu-zesty xmltv-0.6.3/.Dockerfile.ubuntu-zesty --- xmltv-0.6.1/.Dockerfile.ubuntu-zesty 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.Dockerfile.ubuntu-zesty 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -# Dockerfile to generate Ubuntu 17.04 "Zesty Zapus" build container for XMLTV - -FROM ubuntu:zesty - -MAINTAINER Nick Morrott - -ENV TZ 'Europe/London' - -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get -y install \ - libarchive-zip-perl \ - libcgi-pm-perl \ - libdata-dump-perl \ - libdate-calc-perl \ - libdate-manip-perl \ - libdatetime-format-iso8601-perl \ - libdatetime-format-sqlite-perl \ - libdatetime-format-strptime-perl \ - libdatetime-perl \ - libdatetime-timezone-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libfile-chdir-perl \ - libfile-homedir-perl \ - libfile-slurp-perl \ - libfile-which-perl \ - libhtml-parser-perl \ - libhtml-tree-perl \ - libhttp-cache-transparent-perl \ - libhttp-cookies-perl \ - libhttp-message-perl \ - libio-stringy-perl \ - libjson-perl \ - libjson-xs-perl \ - liblingua-preferred-perl \ - liblinux-dvb-perl \ - liblist-moreutils-perl \ - liblog-tracemessages-perl \ - liblwp-protocol-https-perl \ - liblwp-useragent-determined-perl \ - libperlio-gzip-perl \ - libsoap-lite-perl \ - libterm-progressbar-perl \ - libterm-readkey-perl \ - libtimedate-perl \ - libtk-tablematrix-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - libxml-dom-perl \ - libxml-libxml-perl \ - libxml-libxslt-perl \ - libxml-parser-perl \ - libxml-simple-perl \ - libxml-treepp-perl \ - libxml-twig-perl \ - libxml-writer-perl \ - make \ - perl \ - perl-tk \ - tzdata \ - --no-install-recommends - -RUN rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata diff -Nru xmltv-0.6.1/.dockerignore xmltv-0.6.3/.dockerignore --- xmltv-0.6.1/.dockerignore 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.dockerignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -.git -.dockerignore -.Dockerfile* diff -Nru xmltv-0.6.1/filter/tv_imdb xmltv-0.6.3/filter/tv_imdb --- xmltv-0.6.1/filter/tv_imdb 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/filter/tv_imdb 2020-09-07 15:02:53.000000000 +0000 @@ -151,7 +151,7 @@ Needs some more controls for fine tuning "close" matches. For instance, currently it looks like the North America grabber only has date entries for movies, but the imdb.com data contains made for video -movies as well as as real movies, ot is itE<39>s possible to get the +movies as well as as real movies, so itE<39>s possible to get the wrong data to be inserted. In this case we may want to say "ignore tv series" and "ignore tv mini series". Along with this, weE<39>d want to define what a "close" match is. For instance does a movie by the diff -Nru xmltv-0.6.1/filter/tv_to_potatoe xmltv-0.6.3/filter/tv_to_potatoe --- xmltv-0.6.1/filter/tv_to_potatoe 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/filter/tv_to_potatoe 2020-09-07 15:02:53.000000000 +0000 @@ -12,7 +12,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, you can either send email to this # program's maintainer or write to: The Free Software Foundation, -# Inc.; 675 Massachusetts Avenue; Cambridge, MA 02139, USA. +# Inc.; 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. =pod diff -Nru xmltv-0.6.1/.gitignore xmltv-0.6.3/.gitignore --- xmltv-0.6.1/.gitignore 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,24 +0,0 @@ -MYMETA.json -MYMETA.yml -Makefile -blib/ -filter/tv_grep -grab/ch_search/tv_grab_ch_search -grab/fi/tv_grab_fi -grab/huro/tv_grab_huro -grab/it/tv_grab_it -grab/it/tv_grab_it.in2 -grab/it_dvb/tv_grab_it_dvb -grab/na_dd/tv_grab_na_dd -grab/se_swedb/tv_grab_se_swedb -grab/se_tvzon/tv_grab_se_tvzon -grab/uk_bleb/tv_grab_uk_bleb -lib/Supplement.pm -lib/XMLTV.pm -pm_to_blib -tools/tv_validate_file -tools/tv_validate_grabber -t_*_cache -t_*.diff -t_*.log -t_*.xml diff -Nru xmltv-0.6.1/grab/ch_search/tv_grab_ch_search.in xmltv-0.6.3/grab/ch_search/tv_grab_ch_search.in --- xmltv-0.6.1/grab/ch_search/tv_grab_ch_search.in 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/ch_search/tv_grab_ch_search.in 2020-09-07 15:02:53.000000000 +0000 @@ -69,10 +69,11 @@ L. -=head1 AUTHOR +=head1 AUTHORS Daniel Bittel . Inspired by tv_grab_ch by Stefan Siegl. Patric Mueller . +Markus Keller . =head1 BUGS @@ -135,7 +136,6 @@ sub get_channels(); sub channel_id($); sub get_page($); -sub grab_channel($); ## attributes of xmltv root element my $head = { @@ -215,12 +215,19 @@ else { die("never heard of XMLTV mode $mode, sorry :-(") } +## initialize user agent, so that cookie jar can be filled and passed on +my $ua = LWP::UserAgent->new(keep_alive => 300); +my $cookies = HTTP::Cookies->new(); +$ua->cookie_jar($cookies); +$ua->agent("xmltv/$XMLTV::VERSION"); +$ua->env_proxy; + ## hey, we can't live without channel data, so let's get those now! my $bar = new XMLTV::ProgressBar( 'getting list of channels', 1 ) if not $opt_quiet; -my %channels = get_channels(); +my ($secret, %channels) = get_channels(); $bar->update() if not $opt_quiet; $bar->finish() if not $opt_quiet; @@ -321,7 +328,7 @@ ## write out tags -my $paramstr =""; +my $paramstr ="&secret=".$secret; foreach(@requests) { my $id = channel_id($_); my %channel = ('id' => $id, @@ -341,17 +348,11 @@ my $url=$head->{q(source-data-url)}; -my $ua = LWP::UserAgent->new(keep_alive => 300); -$ua->cookie_jar(HTTP::Cookies->new()); -$ua->agent("xmltv/$XMLTV::VERSION"); -$ua->env_proxy; - my $req = HTTP::Request->new(POST => $url); $req->content_type('application/x-www-form-urlencoded'); $req->content(substr ( $paramstr, 1)); -# FIXME what is this request doing? It fills the cookie jar -$ua->request($req); +# store the selected channels $ua->request($req); ## write out tags @@ -557,10 +558,15 @@ my $url=$head->{q(source-data-url)}; my $tb=new HTML::TreeBuilder(); - $tb->parse(get_page($url)) + # For some reason, we need to fetch this page twice. Probably a bug in search.ch. + # If you open https://tv.search.ch/channels as first page in private mode in Firefox, the first click on the Save button doesn't work either. + $ua->get($url); + $tb->parse($ua->get($url)->content) or die "cannot parse content of $url"; $tb->eof; + my $secret = ($tb->look_down('name', 'secret'))->attr('value'); + ## getting the channels directly selectable foreach($tb->look_down('_tag' => 'label')) { my $id = ($_->look_down('_tag' => 'input'))->id; # tv-channel-sf1 @@ -571,7 +577,7 @@ } $tb->delete; - return %channels; + return ($secret, %channels); } diff -Nru xmltv-0.6.1/grab/combiner/test.conf xmltv-0.6.3/grab/combiner/test.conf --- xmltv-0.6.1/grab/combiner/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/combiner/test.conf 2020-09-07 15:02:53.000000000 +0000 @@ -1,2 +1,2 @@ -grabber=/usr/bin/tv_grab_se_swedb;root-url=http://xmltv.tvsajten.com/channels.xml.gz&ncachedir=/tmp/.xmltv/cache&nchannel=dr1.dr.dk -grabber=/usr/bin/tv_grab_eu_dotmedia;country=Austria&ncachedir=/tmp/.xmltv/cache&nchannel=arte.de +grabber=/usr/bin/tv_grab_uk_bleb;bbc1 +grabber=/usr/bin/tv_grab_uk_bleb;bbc2 diff -Nru xmltv-0.6.1/grab/dtv_la/test.conf xmltv-0.6.3/grab/dtv_la/test.conf --- xmltv-0.6.1/grab/dtv_la/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/dtv_la/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -country VE Venezuela https://www.directv.com.ve/ -channel 102 Venevisión (102) -channel 105 Canal i (105) diff -Nru xmltv-0.6.1/grab/dtv_la/tv_grab_dtv_la xmltv-0.6.3/grab/dtv_la/tv_grab_dtv_la --- xmltv-0.6.1/grab/dtv_la/tv_grab_dtv_la 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/dtv_la/tv_grab_dtv_la 1970-01-01 00:00:00.000000000 +0000 @@ -1,1106 +0,0 @@ -#!/usr/bin/perl -=pod - -=head1 NAME - -tv_grab_dtv_la - Grab TV listings for Direct TV Latin America - -=head1 SYNOPSIS - -tv_grab_dtv_la --help - -tv_grab_dtv_la [--config-file FILE] --configure [--gui OPTION] - -tv_grab_dtv_la [--config-file FILE] [--output FILE] [--days N] - [--offset N] [--min-delay N] [--max-delay N] [--quiet] - -tv_grab_dtv_la --list-channels --loc [ar | cb | cl | co | ec | pe | pr | uy | ve] - -tv_grab_dtv_la --capabilities - -tv_grab_dtv_la --version - -=head1 DESCRIPTION - -Output TV listings for Direct TV channels available in Latin America. -Listings for the following countries are currently available: -Argentina, Caribbean ('cb'), Chile, Colombia, Ecuador, -Peru, Puerto Rico, Trinidad, Uruguay, Venezuela. - -The TV listings come from http://directstage.directvla.com/ -The grabber relies on parsing HTML so it might stop working at any time. - -First run B to choose, first of all your country -and then which channels you want to download. Then running B -with no arguments will output listings in XML format to standard output. - -The grabber doesn't generate stop times, so you may want to run -tv_sort on the output to generate them. - -B<--configure> Prompt for which channels, and write the configuration file. - -B<--config-file FILE> Set the name of the configuration file, the -default is B<~/.xmltv/tv_grab_dtv_la.conf>. This is the file written by -B<--configure> and read when grabbing. - -B<--gui OPTION> Use this option to enable a graphical interface to be used. -OPTION may be 'Tk', or left blank for the best available choice. -Additional allowed values of OPTION are 'Term' for normal terminal output -(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar. - -B<--output FILE> Write to FILE rather than standard output. - -B<--days N> Grab N days. The default is 3. - -B<--offset N> Start N days in the future. The default is to start -from today. - -B<--min-delay N> You must insert a delay between page requests to avoid -unnecessary load on the website. If you try to grab pages too quickly then -it's likely you will get banned by the website providers (and may get -all other xmltv users banned as well!). -Suggested value: 1 (seconds) - -B<--max-delay N> Maximum delay between web page fetches. -Suggested value: 3 (seconds) - -B<--quiet> Suppress the progress messages normally written to standard -error. - -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L - -B<--version> Show the version of the grabber. - -B<--help> Print a help message and exit. - -=head1 SEE ALSO - -L, L. - -=head1 AUTHOR - -Most of the grabber was made by Lic. Christian A. Rodriguez , with a -lot of help from others, specially Joaquin Salvarredy who -tested the grabber from its early versions and Lic. Nicolas Macia - -=head1 BUGS - -This grabber extracts all information from Direct TV Latin America website. Any change in this -web page may cause this grabber to stop working. - -=cut - -# Author's TODOs & thoughts -# -# Add better channel names -# -( #(facilitate code-folding) -# -# 2016-03-14 -# -# URLS -# http://www.directv.com.ar/ -# https://www.directv.com.ar/programacion/guia-de-programacion -# http://www.directv.com.ar/programacion/guia-de-canales -# -# http://www.directv.cl/ -# https://www.directv.cl/guia/guia.aspx?type=&link=nav/ -# http://www.directv.cl/planes/guia-de-canales -# -# http://www.directv.com.co/ -# https://www.directv.com.co/guia/guia.aspx?type= -# http://www.directv.com.co/paquetes/guia-de-canales -# -# http://www.directv.com.ec/ -# https://www.directv.com.ec/guia/guia.aspx?type= -# http://www.directv.com.ec/planes/guia-de-canales -# -# http://www.directv.com.pe/ -# https://www.directv.com.pe/guia/guia.aspx?type= -# http://www.directv.com.pe/paquetes/guia-de-canales -# -# http://www2.directvpr.com/ -# https://www.directvpr.com/guia/guia.aspx?type=&link=nav -# http://www.directvpr.com/guia-de-canales?link=nav - -# http://www.directv.com.uy/ -# https://www.directv.com.uy/guia/guia.aspx?type= -# http://www.directv.com.uy/paquetes/guia-de-canales -# -# http://www.directv.com.ve -# https://www.directv.com.ve/guia/guia.aspx -# http://www.directv.com.ve/planes/guia-de-canales -# -); - - -###################################################################### -## REQUIRED LIBRARIES -###################################################################### -use warnings; -use strict; - -use XMLTV; -use XMLTV::Version "$XMLTV::VERSION"; -use XMLTV::Capabilities qw/baseline manualconfig/; -use XMLTV::Description 'Latin America Direct TV listings'; -use XMLTV::Memoize; -use XMLTV::ProgressBar; -use XMLTV::Ask; -use XMLTV::Config_file; -use XMLTV::Mode; -use XMLTV::Date; -use XMLTV::DST; -use XMLTV::Usage < 'http://directstage.directvla.com/', - 'source-data-url' => 'http://directstage.directvla.com/', - 'generator-info-name' => 'tv_grab_dtv_la', - 'generator-info-url' => 'http://xmltv.org/', - }; - -my $channels_icon_url="http://www.lyngsat.com/packages/directvlatin.html"; -my $countries_list_url="http://directstage.directvla.com/"; - -# So we are not affected by winter/summer timezone -$XMLTV::DST::Mode='none'; - -# timezone to use (for all countries!) -my $TZ="-0300"; - -# default language -my $LANG="es"; -my $OUT_ENCODING="UTF-8"; - -# Selected country -my %country; - -# Full list of channels -my @ch_all; -my $CHANNELS_URL=undef; - -# Providers name for creating unique channel id -my $PROVIDER_NAME="dtv.la"; - -# Progressbar -my $mainbar; - -# Private UserAgent -my $cookies = HTTP::Cookies->new; -my $ua = LWP::UserAgent->new; -$ua->cookie_jar($cookies); - -$ua->agent("xmltv/$XMLTV::VERSION"); -$ua->parse_head(0); -$ua->env_proxy; - - -# undocumented --cache option. -# not sure this will work with ajax post requests ? -XMLTV::Memoize::check_argv('get_tree'); - -###################################################################### -## SUBROUTINES -###################################################################### - -###################################################################### -## Returns a trimmed string -sub trim { - my $string = shift; - $string =~ s/^\s+|\s+$//g if defined $string; - return $string; -} - -###################################################################### -## Returns a TreeBuilder instance - -# You must insert a delay between page requests to avoid -# unnecessary load on the website. If you try to grab pages too quickly then -# it's likely you will get banned by the website providers (and may get -# all other xmltv users banned as well - it's trivial to ban by user-agent string). -# -my $last_get_time; -# -sub get_tree ($;$$) { - my $url = shift; - my $method = shift || 'get'; - my $data = shift; - my $r; - - print STDERR "$method: $url ".($data?"[$data]":'')." \n" if $opt_debug; - - # let's not overload the website with too many requests so we'll restrict the request frequency (as per Get_nice) - - my $Delay = $opt_max_delay - $opt_min_delay; # in seconds - my $MinDelay = $opt_min_delay; # in seconds - - if (defined $last_get_time) { - # A page has already been retrieved recently. See if we need - # to sleep for a while before getting the next page - being - # nice to the server. - my $next_get_time = $last_get_time + (rand $Delay) + $MinDelay; - my $sleep_time = $next_get_time - time(); - sleep $sleep_time if $sleep_time > 0; - } - - if (!defined $method || lc($method) eq 'get') { - $r = $ua->get($url); - - } elsif (lc($method) eq 'post') { - $r = $ua->post($url, $data); # $data must be a hash - - } elsif (lc($method) eq 'jsonpost') { - # - # create the http request - my $req = HTTP::Request->new( 'POST', $url ); - ##$req->header( 'Content-Type' => 'application/json' ); - $req->content_type( 'application/json; charset=utf-8' ); - $req->content( $data ); # data must be json - - # execute the request - $r = $ua->request($req); - - } else { - die "unknown fetch method '$method'"; - } - $last_get_time = time(); - - #print STDERR Dumper($r);die; - die "Could not fetch $url". (lc($method) eq 'jsonpost'?"[$data]":'') .", error: " . $r->status_line if ($r->is_error); - - my $t; - if (lc($method) eq 'jsonpost') { - # expect a json reply! - $t = JSON->new()->utf8(1)->decode($r->content) or die "cannot parse content of $url\n"; - - } else { - $t = new HTML::TreeBuilder; - #$t->utf8_mode(1); - $data=$r->decoded_content('default_charset'=>'utf8'); - #$data=decode('UTF-8',$data) if (is_utf8($data)); - $t->parse($data) or die "Cannot parse content of Tree\n"; - $t->eof; - } - return $t; -} - -###################################################################### -## Bump a YYYYMMDD date by one. -sub nextday { - my $d = shift; - my $p = parse_date($d); - my $n = DateCalc($p, '+ 1 day'); - return UnixDate($n, '%Q'); -} - -###################################################################### -## Returns the URL for grabbing channels -sub get_channels_url { - if (not defined $CHANNELS_URL){ - die "No country specified, run me with --configure\n" if not keys %country; - - # as at Apr 2014 it looks like they're still working on the website: all the Caribbean channels - # point to the same place. - if ( $country{'id'} eq 'CB' ) { - $CHANNELS_URL = "http://www.directvcaribbean.com/tt/channel-description"; - } - else { - # although some of the sites have this as a subdir (e.g. under 'paquetes' or 'planes') it still - # seems to work without that - $CHANNELS_URL = $country{url} . 'guia-de-canales'; - } - - } - - return $CHANNELS_URL; -} - -###################################################################### -## Returns the URL for grabbing specified channel programs -sub get_channel_programs_url($) { - ##my $ch_id=shift; - ##my $base_url=get_channels_url(); - ##$base_url=~ s/default/detailch/; - ##return "$base_url?c=$ch_id&n=chname"; - - # e.g. http://www.directv.com.ar/guia/Services/ProgrammingGuideAjax.asmx/GetProgramming - - return $country{'url'} . 'guia/Services/ProgrammingGuideAjax.asmx/GetProgramming'; -} - -###################################################################### -## Returns the URL for grabbing programme details -sub get_program_detail_url() { - - # e.g. http://www.directv.com.ar/guia/Services/ProgrammingGuideAjax.asmx/GetProgrammingDetail - - return $country{'url'} . 'guia/Services/ProgrammingGuideAjax.asmx/GetProgrammingDetail'; -} - -###################################################################### -## Converts the given datetime format to the needed UTC format -sub datetime_for_program( $;$ ){ - my ($date,$strdt)=@_; - $strdt=~ /^(\w*)\s+(\d{1,2}:\d{1,2})/; - if ( defined $1 and defined $2) { - my $weekday=$1; - my $time=UnixDate($2,"%H:%M"); - if ( UnixDate($date,"%a") eq $weekday ){ - return utc_offset("$date $time", $TZ) - } - } - return undef; -} - -###################################################################### -## Returns channel programs for the specified date and channel id -sub get_channel_programs ( $$$$ ) { - my ($ref_dates, $ref_channels, $ref_ch_all, $ref_programmes) = @_; - - # convert @dates & @channels into hashes for faster searching - my %_dates = map { $_ => 1 } @$ref_dates; - my %_channels = map { $_ => 1 } @$ref_channels; - - my @_ch_all = @$ref_ch_all; - - # temporary store for programmes we fetch (used for detecting duplicates and clumps) - my $programmes = {}; - - # for parsing non-English language dates - my $lang; - if ( $country{'id'} eq 'CB' ) { # Caribe is currently in English - $lang = Date::Language->new('English'); - } else { - $lang = Date::Language->new('Spanish'); - } - - # site now uses a jQuery AJAX POST with JSON content in UTF-8 - # e.g. { "day":19, "time":"12","minute":"30", "month":"4", "year":"2014", "onlyFavorites":"N" } - # - # data are avialable in a 4-hour windows (all channels combined) - # - foreach my $date (@$ref_dates) { - my ( $_y, $_m, $_d ) = $date =~ /(\d\d\d\d)(\d\d)(\d\d)/; - for (my $i=0; $i<24; $i+=4) { - my $_h = substr("0$i", -2, 2); - - my $data = '{ "day":'.$_d.', "time":"'.$_h.'","minute":"00", "month":"'.$_m.'", "year":"'.$_y.'", "onlyFavorites":"N" }'; - ##print STDERR $data."\n"; - - - my $json = get_tree( get_channel_programs_url(undef), 'jsonpost', $data ); - ##print STDERR Dumper($json);die; - - # response is a JSON string containing just one k:v pair, 'd' => "..." - # (see http://www.directv.com.ar/guia/js/Program-Guide/ProgrammingGuideAjax.js for details) - - # we don't need the overhead of TreeBuilder - we'll go 'old school' and use a regexp - my (@li) = $json->{'d'} =~ m/()/g; - ##print Dumper(@li);die; - - foreach my $li (@li) { - - #
  • Las aventuras de Robin Hood
    • - #
  • Enciclopedia di...
    • - - #
  • Programación no disponible
    • - - # Method: - # The programme schedule is returned as a 4-hour window. Unfortunately the html contains nothing of use - # other than the title, channel id and eventId. There isn't even a start time! So: - # 1. Extract all the
  • items - # 2. Ignore any which aren't for a requested channel - # 3. Ignore any which have already started (as they will have already been picked up in a previous 4-hour window) - # 4. Fetch the programme details using the eventId - # 5. Parse the prog details and add to a hash - # - - # parse the
  • element - my ( $eventId, $channelId, $hasStarted ) = $li =~ m/eventId="(\d*)"\schannel="(\d*)".*?(?(?=.*ChannelArrowLeft)(ChannelArrowLeft)|())/; - ##if ($hasStarted) {print STDERR "skipping $eventId, $channelId\n";} - next if $hasStarted; # if prog has already started - next if ! $_channels{ $channelId }; # if channel not wanted - next if !defined $eventId || $eventId eq '';# e.g. Programación no disponible (can't output anything since no start/stop time!) - - # post content: { "eventId":121190335202, "day":20, "time":"4","minute":"0", "month":"4", "year":"2014" } - my $data = '{ "eventId":'.$eventId.', "day":'.$_d.', "time":"'.$_h.'","minute":"0", "month":"'.$_m.'", "year":"'.$_y.'" }'; - - my $json = get_tree( get_program_detail_url(), 'jsonpost', $data ); - ##print STDERR Dumper($json);die; - - my $t = HTML::TreeBuilder->new()->parse( $json->{'d'} ) or die "cannot parse content of programme detail\n"; - $t->eof; - ##$t->dump();die; - - my $p; # programme - - my $div; if ( my $_t = $t->look_down('_tag'=>'h2') ) { $div = $_t->parent(); } - if (!defined $div) { - # why is it not? - print STDERR 'Warn: No programme description (no

    element for "eventId":'.$eventId.', "day":'.$_d.', "time":"'.$_h.")\n"; - next; - } - - my $h; # html (tree) element - - if ( $h = $div->look_down('_tag'=>'h2') ) { - if ( my $h_ = $div->look_down('_tag'=>'img', 'alt'=>'HD program') ) { - $p->{'video'}->{'quality'} = 'HDTV'; - } - $p->{'title'} = trim( $h->as_text() ); - $h->detach; - } - # 'title' is mandatory in the DTD. If we don't have one then set to unknown - $p->{'title'} = ($LANG eq 'pt_BR' ? 'ignorado' : 'incógnito') if (!defined $p->{'title'} || $p->{'title'} eq ''); - - - # 1st

    is the description - if ( $h = $div->look_down('_tag'=>'p') ) { - $p->{'desc'} = trim( $h->as_text() ); - $h->detach; - } - - # 2nd

    is the start time and duration - if ( $h = $div->look_down('_tag'=>'p') ) { - my $h_ = trim( $h->as_text() ); - my ($_junk, $_date, $_dur) = $h_ =~ m/(Comienza|Start):\s*(.*?)\|(.*?)$/s; # Caribe = "Start:" - - # Date::Language doesn't currently do Portuguese - # (the Sky BR site isn't handled in this grabber anyway) - my $dt; - if ( $country{'id'} eq 'BR' ) { - die "\n Sorry I don't speak Portuguese \n"; - } else { - $dt = $lang->str2time($_date, $TZ); - } - - $p->{'start_epoch'} = $lang->str2time($_date, $TZ); - ( $p->{'duration'} ) = $_dur =~ /(\d*)\s(?=minutos|minutes)/; - $p->{'stop_epoch'} = $p->{'start_epoch'} + ( $p->{'duration'} * 60 ) if $p->{'duration'}; - $p->{'start'} = $lang->time2str( "%Y%m%d%H%M%S %z", $p->{'start_epoch'}, $TZ ); - $p->{'stop'} = $lang->time2str( "%Y%m%d%H%M%S %z", $p->{'stop_epoch'}, $TZ ); - $h->detach; - } - - #

    class "Rank" = rating & programme url - if ( $h = $div->look_down('_tag'=>'div', 'class'=>qw/Rank/) ) { - if ( my $h = $h->look_down('_tag'=>'p') ) { - my $h_ = trim( $h->as_text() ); - ( $p->{'rating'} ) = $h_ =~ m/Rating:\s*(\S*)\s/s; - } - - if ( my $h = $h->look_down('_tag'=>'div') ) { - if ( my $h_ = $h->look_down('_tag'=>'a') ) { - my $h__ = trim( $h_->attr('href') ); - $h__ = $country{'url'} . $h__ if ( $h__ !~ /^http/ ); - $p->{'url'} = $h__; - } - } - $h->detach; - } - - - - # Reformat the data to Create the data structure for the programme - my $p_out = {}; - $p_out->{'channel'} = $channelId . '.' . $PROVIDER_NAME; - $p_out->{'title'} = [[ encode($OUT_ENCODING, $p->{'title'}), $LANG ]]; - $p_out->{'start'} = $p->{'start'}; - $p_out->{'stop'} = $p->{'stop'} if (defined $p->{'stop'} && $p->{'stop'} ne ''); - $p_out->{'desc'} = [[ encode($OUT_ENCODING, $p->{'desc'}), $LANG ]] if (defined $p->{'desc'} && $p->{'desc'} ne ''); - $p_out->{'sub-title'} = [[ encode($OUT_ENCODING, $p->{'sub_title'}), $LANG ]] if (defined $p->{'sub_title'} && $p->{'sub_title'} ne ''); - $p_out->{'rating'} = [[ $p->{'rating'} ]] if (defined $p->{'rating'} && $p->{'rating'} ne ''); - $p_out->{'url'} = [ encode($OUT_ENCODING, $p->{'url'}) ] if (defined $p->{'url'} && $p->{'url'} ne ''); - $p_out->{'video'} = $p->{'video'} if (defined $p->{'video'}); - - # store the programme avoiding duplicates - # also check for duplicate start times and set clumpidx - if ( defined $programmes->{ $channelId }->{ $p->{'start_epoch'} } ) { - # duplicate prog or contemporary? - my $dup = 0; - foreach my $_p ( @{ $programmes->{ $channelId }->{ $p->{'start_epoch'} } } ) { - $dup = 1 if ( $_p->{'title'}[0][0] eq $p_out->{'title'}[0][0] ); # duplicate - } - next if $dup; # ignore duplicates (go to next
  • programme element) - if (!$dup) { - # contemporary programme so set clumpidx - my $numclumps = scalar @{ $programmes->{ $channelId }->{ $p->{'start_epoch'} } } + 1; - # set (or adjust) clumpidx of existing programmes - my $i = 0; - foreach my $_p ( @{ $programmes->{ $channelId }->{ $p->{'start_epoch'} } } ) { - $_p->{'clumpidx'} = "$i/$numclumps"; - $i++; - } - # set clumpidx for new programme - $p_out->{'clumpidx'} = "$i/$numclumps"; - } - } - - # store the programme in our temporary store - push @{ $programmes->{ $channelId }->{ $p->{'start_epoch'} } }, $p_out; - - } - - $mainbar->update() if not $opt_quiet; - } - - } - - - # All data has been gathered. We can now write the programme element to the output. - # - foreach ( keys %{$programmes} ) { - my $_ch_progs = $programmes->{$_}; - foreach ( sort keys %{$_ch_progs} ) { - my $_dt_progs = $_ch_progs->{$_}; - foreach (@{ $_dt_progs }) { - push @{$ref_programmes}, $_; - } - } - } -} - -###################################################################### -## Returns the list of channels -# -# Note: I've noticed that sometimes there's more channels on the actual programme schedule page -# than in the channels guide page :-( So we may need to switch and get the list of channels -# from the AJAX fetch on the schedules page (although the icons may be smaller?) -# -sub get_channels { - my $bar = new XMLTV::ProgressBar("Getting list of channels for $country{name}", 1) if not $opt_quiet; - - my %channels; - my $url=get_channels_url(); - - # Get channels that are transmiting now - my $tree = get_tree($url); - get_channels_from_tree($tree,\%channels); - # We will try to find more channels for later hours - #get_channels_for_later_hours($tree,\%channels); - - # Finish using Tree - $tree=undef; - $bar->update() && $bar->finish() if not $opt_quiet; - return %channels; -} - -###################################################################### -## Simulate a form filling to retrieve more channels for later hours -sub get_channels_for_later_hours() { - my ($tree,$channels) = @_; - - # First we get the form elemento to call iteratively for each option from a select - my $form_elem = $tree->look_down( - "_tag"=>"form", sub { - defined $_[0]->attr('name') and $_[0]->attr('name')=~ /Form1/i - } - ); - # The name of the select element is: - my $search_for_input="ddlTime"; - my %needed_form_elems=('ddlTime','select','ddlDay','select','btnSubmit','input'); - - # Form to call iteratively - my $form=HTML::Form->parse($form_elem->as_HTML(),get_channels_url()); - my $input; - - foreach my $ninput (keys %needed_form_elems){ - $input=$form->find_input($ninput); - - # There is a bug in the source HTML. The field we need is outside the form tag - if (not defined $input) { - # We try to fix this problem - my $broken_elem = $tree->look_down( - "_tag"=>$needed_form_elems{$ninput}, sub { - defined $_[0]->attr('name') and $_[0]->attr('name')=~ /$ninput/i - } - ); - $form_elem->insert_element($broken_elem); - $form=HTML::Form->parse($form_elem->as_HTML(),get_channels_url()); - $input=$form->find_input($ninput); - die "Cannot retrieve field $ninput. Aborting" if (not defined $input); - } - } - # Now for each value of the select, we will call get_channels_from_tree subroutine - $input=$form->find_input($search_for_input); - my $default_value=$input->value; - foreach ($input->possible_values) { - if ($_ != $default_value) { - $form->value($search_for_input,$_); - my $r=$ua->request($form->click); - die "Error doing automatic form filling. Aboring" if ($r->is_error); - my $t = new HTML::TreeBuilder; - #$t->utf8_mode(1); - my $data=$r->decoded_content('default_charset'=>'utf8'); - #$data=from_to($data,'UTF-8',$OUT_ENCODING) if (is_utf8($data)); - $t->parse($data) or die "Cannot parse content of Tree\n"; - $t->eof; - get_channels_from_tree($t,$channels); - } - } -} - -###################################################################### -## Return the list of channels for a tree representation of an HTML page -sub get_channels_from_tree( ) { - my ($tree,$channels) = @_; - - # see if there's a 'pMain' so we can ignore the CMS content (which contains the on-demand channels) - my $chan_div = $tree->look_down('_tag' => 'div', 'id' => 'pMain'); - $tree = $chan_div if $chan_div; - - my @chan_groups = $tree->look_down('_tag' => 'div', 'class' => 'guia-canales')->look_down('_tag' => 'div', 'class' => 'combo-canal-content'); - - foreach (@chan_groups) { - my @chan_elems = $_->look_down('_tag' => 'li'); - foreach (@chan_elems) { - #
  • - # TVPeru
    - # 197 - #     - #
    • - - if ( my $chan = $_->look_down('_tag' => 'a') ) { - my ($chan_id, $chan_name, $chan_url, %chan_icon) = ('', '', '', ()); - $chan_id = trim( $chan->look_down('_tag' => 'span')->as_text() ); - $chan_url = $chan->attr('href'); - if ( my $chan_img = $chan->look_down('_tag' => 'img') ) { - $chan_name = trim( $chan_img->attr('alt') ); - $chan_icon{'src'} = $chan_img->attr('src'); - $chan_icon{'width'} = $chan_img->attr('width') if defined $chan_img->attr('width'); - $chan_icon{'height'} = $chan_img->attr('height') if defined $chan_img->attr('height'); - } - - $chan_name="$chan_name ($chan_id)"; - if (not exists ${$channels} { $chan_id }) { - ${$channels} {$chan_id}=$chan_name; - push @ch_all, { - 'display-name' => [[ encode("UTF-8",$chan_name), $LANG ],[$chan_id]], - 'channel-num' => $chan_id , - 'id' => "$chan_id.$PROVIDER_NAME", - 'icon' => [ \%chan_icon ], - }; - } - } - } - } - -} - -###################################################################### -## Get a list of possible countries -sub get_countries( ) { - my $country_codes = { 'Argentina' => 'AR' - , 'Caribe' => 'CB' - , 'Chile' => 'CL' - , 'Colombia' => 'CO' - , 'Ecuador' => 'EC' - , 'Perú' => 'PE' - , 'Puerto Rico' => 'PR' - , 'Uruguay' => 'UY' - , 'Venezuela' => 'VE' - }; - - my $tree = get_tree($countries_list_url); - my @options=$tree->look_down('_tag' => 'div', 'class' => 'box-menu')->look_down('_tag' => 'div', 'class' => 'items')->look_down('_tag' => 'a'); - my %countries; - foreach my $tag (@options){ - my %country; - $country{'name'} = $tag->as_text(); - $country{'url'} = $tag->attr('href') . "/"; - # Default URLs to https:// - programme guide is https; channel lists will redirect to http - $country{'url'} =~ s/^http:/https:/; - $country{'id'} = $country_codes->{$country{'name'}}; - - # we won't do the Sky sites - they are very different to DirecTV - if ( $country{'name'} =~ /(SKY Brazil|SKY México)/ ) { - #print "Skipping country - $country{'name'} \n" unless $opt_quiet; - next; - } - - if ( !defined $country_codes->{$country{'name'}} ) { - print "Unknown country - $country{'name'} \n" unless $opt_quiet; - next; - } - - $countries{$tag->as_text()} = \%country; - } - return %countries; -} - -###################################################################### -## Return the user-selected country -sub select_country( ) { - my %countries = get_countries(); - my @names = sort keys %countries; - my $choice = ask_choice("Select your country:", $names[0], @names); - return ( id=>$countries{$choice}{'id'}, name=>$choice, url=>$countries{$choice}{'url'} ); -} - -###################################################################### -## Return the channel icons from LyngSat -sub get_channel_icons() { - my $bar = new XMLTV::ProgressBar("Trying to fetch channel icons for $country{name}", $#ch_all + 1) if not $opt_quiet; - my $tree=get_tree($channels_icon_url); - my $table=$tree->look_down( - '_tag'=>'table',sub { - defined $_[0]->attr('width') and $_[0]->attr('width')== '600' - } - ); - foreach my $ch (@ch_all){ - my $ch_num=$ch->{'channel-num'}; - my $tr=$table->look_down( - '_tag'=>'tr',sub { - my @td=$_[0]->content_list(); - defined $td[0] and $td[0]->as_text() =~ /\s*$ch_num\s*/ - } - ); - if (defined $tr){ - my $img=$tr->look_down( - '_tag'=>'img'); - $ch->{icon}=[ { src=>$img->attr('src')} ] if defined $img and defined $img->attr('src'); - } - $bar->update() if not $opt_quiet; - } - $bar->finish() if not $opt_quiet; -} - -###################################################################### -## Return the channel icons from the DirecTV site -sub get_channel_icons_dtv() { - my $bar = new XMLTV::ProgressBar("Fetching channel icons for $country{name}", $#ch_all + 1) if not $opt_quiet; - my $tree=get_tree( get_channels_url() ); - my $table=$tree->look_down('_tag' => 'div', 'class' => 'guia-canales'); - - foreach my $ch (@ch_all){ - my $ch_num=$ch->{'channel-num'}; - my $chan_img; - if ( my $chan_a = $table->look_down('_tag'=>'a', 'href'=>qr/ChannelDetail.aspx\?id=$ch_num/) ){ - $chan_img = $chan_a->look_down('_tag'=>'img'); - } - if (defined $chan_img) { - my %chan_icon; - $chan_icon{'src'} = $chan_img->attr('src'); - $chan_icon{'width'} = $chan_img->attr('width') if defined $chan_img->attr('width'); - $chan_icon{'height'} = $chan_img->attr('height') if defined $chan_img->attr('height'); - $ch->{icon}=[ \%chan_icon ]; - } - $bar->update() if not $opt_quiet; - } - $bar->finish() if not $opt_quiet; -} - -###################################################################### -## MAIN PROGRAM -###################################################################### - -###################################################################### -## get options -# Get options. - -$opt_days = 3; # default -$opt_offset = 0; # default -$opt_quiet = 0; # default -$opt_min_delay = 1; -$opt_max_delay = 3; -$opt_debug = 0; - -GetOptions( - 'days=i' => \$opt_days, - 'offset=i' => \$opt_offset, - 'help' => \$opt_help, - 'configure' => \$opt_configure, - 'config-file=s' => \$opt_config_file, - 'gui:s' => \$opt_gui, - 'output=s' => \$opt_output, - 'quiet' => \$opt_quiet, - 'list-channels' => \$opt_list_channels, - 'debug' => \$opt_debug, - 'loc=s' => \$opt_loc, - 'min-delay=f' => \$opt_min_delay, - 'max-delay=f' => \$opt_max_delay, -) or usage(0); - -$opt_min_delay = (0.5, $opt_min_delay)[0.5 < $opt_min_delay]; -$opt_max_delay = (0.5, $opt_max_delay)[0.5 < $opt_max_delay]; - -die 'number of days must not be negative' if (defined $opt_days && $opt_days < 0); -usage(1) if $opt_help; - -XMLTV::Ask::init($opt_gui); -my $mode = XMLTV::Mode::mode( - 'grab', # default - $opt_configure => 'configure', - $opt_list_channels => 'list-channels', -); - -# File that stores which channels to download. -my $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_dtv_la', $opt_quiet); -my @config_lines; # used only in grab mode -if ($mode eq 'configure') { - XMLTV::Config_file::check_no_overwrite($config_file); -}elsif ($mode eq 'grab') { - @config_lines = XMLTV::Config_file::read_lines($config_file); -}elsif ($mode eq 'list-channels') { - # Config file not used. -}else { - die -} - -## Whatever we are doing, we need the channels data. -##my %channels = get_channels(); # sets @ch_all -my %channels; -my @channels; - -###################################################################### -## write configuration -# -if ($mode eq 'configure') { - open(CONF, ">$config_file") or die "cannot write to $config_file: $!"; - %country= select_country(); - print CONF "country $country{id} $country{name} $country{url} \n"; - %channels = get_channels(); # sets @ch_all - - # Ask about each channel. - my @chs = sort keys %channels; - my @names = map { $channels{$_} } @chs; - my @qs = map { "add channel $_?" } @names; - my @want = ask_many_boolean(1, @qs); - foreach (@chs) { - my $w = shift @want; - warn("cannot read input, stopping channel questions"), last - if not defined $w; - # No need to print to user - XMLTV::Ask is verbose enough. - - # Print a config line, but comment it out if channel not wanted. - print CONF '#' if not $w; - my $name = shift @names; - print CONF "channel $_ $name\n"; - # TODO don't store display-name in config file. - } - close CONF or warn "cannot close $config_file: $!"; - say("Finished configuration."); - exit(); -} - -# Not configuration, we must be writing something, either full -# listings or just channels. - -die if $mode ne 'grab' and $mode ne 'list-channels'; - -# Options to be used for XMLTV::Writer. -my %w_args; -if (defined $opt_output) { - my $fh = new IO::File(">$opt_output"); - die "cannot write to $opt_output: $!" if not defined $fh; - $w_args{OUTPUT} = $fh; -} -$w_args{encoding} = $OUT_ENCODING; -my $writer = new XMLTV::Writer(%w_args); -$writer->start($HEAD); - -if ($mode eq 'list-channels') { - # must have a country before we can list channels! - die "please select a country ('--loc xx')" if (!defined $opt_loc || $opt_loc eq ''); - - my %countries = get_countries(); - my ($key, $value); - while ( ($key, $value) = each %countries ) { - undef $key; - if ( $value->{'id'} eq uc($opt_loc) ) { - %country = ( id => $value->{'id'}, name => $value->{'name'}, url => $value->{'url'} ); - last; - } - } - - %channels = get_channels(); # sets @ch_all based on %country - - foreach (@ch_all) { - delete $_->{'channel-num'}; # not an DTD item! - $writer->write_channel($_) ; - } - $writer->end(); - exit(); -} - - -###################################################################### -## We are producing full listings. -die if $mode ne 'grab'; - -## Read configuration -# @channels = id list of channels to grab -# %channels = id => name of channels to grab -# @ch_all = id + ch-num + display-name of channels to grab -# -my $line_num = 1; -foreach (@config_lines) { - ++ $line_num; - next if not defined; - if (/^country:?\s+(\S+)\s+(\S+)\s+([^\#]+)/) { - %country=( id => $1, name=>$2, url=>$3 ); - }else{ - if (/^channel:?\s+(\S+)\s+([^\#]+)/) { - my $ch_did = $1; - my $ch_name = $2; - $ch_name =~ s/\s*$//; - push @channels, $ch_did; - #CAR - push @ch_all, { - 'display-name' => [[ $ch_name, $LANG ],[$ch_did]], - 'channel-num' => $ch_did , - 'id'=> "$ch_did.$PROVIDER_NAME" }; - $channels{$ch_did} = $ch_name; - } else { - warn "$config_file:$line_num: bad line\n"; - } - } -} - -###################################################################### -## begin main program -## Assume the listings source uses CET (see BUGS above). -my $now = DateCalc(parse_date('now'), "$opt_offset days"); - -die "No channels specified, run me with --configure\n" if not keys %channels; -die "No country specified, run me with --configure\n" if not keys %country; -my @to_get; - -## we change language if country is Brazil -$LANG="pt_BR" if $country{name} =~ /brazil/i; - -# Dates requested for programs listing -# @dates = list of dates to grab (yyyymmdd) -# -my $day=UnixDate($now,'%Q'); -my @dates; -for (my $i=0;$i<$opt_days;$i++) { - push @dates, $day; - #for each day - $day=nextday($day); - die if not defined $day; -} - -# Try to get channel icons -# adds %icon to @ch_all -# -##get_channel_icons(); # LyngSat -get_channel_icons_dtv(); # DirecTV - -# Write the elements -# data from %channels -# @to_get = array of @dates (yyyymmdd), chan-id (e.g. 122), chan-name (e.g. 122.dtv.la) -# -foreach my $ch_did (@channels) { - my $index=0; - my $ch_name=$channels{$ch_did}; - my $ch_xid="$ch_did.$PROVIDER_NAME"; - while (${$ch_all[$index]}{'id'} ne $ch_xid) { - $index=$index+1; - } - my $ch_num=${ch_all[$index]}{'channel-num'}; - my $to_add={ - id => $ch_xid, - 'display-name' => [ - [ encode($OUT_ENCODING, $ch_name), $LANG ], - [ $ch_num ] ] - }; - $to_add->{icon}=${ch_all[$index]}{icon} if (exists ${ch_all[$index]}{icon} ); - $writer->write_channel($to_add); - # build array of station-days to grab - push @to_get, [ \@dates, $ch_xid, $ch_num ]; -} - -# This progress bar is for both downloading and parsing. Maybe -# they could be separate. -##my $mainbar = new XMLTV::ProgressBar("getting listings for $country{name}", $#to_get + 1) if not $opt_quiet; -$mainbar = new XMLTV::ProgressBar("Getting listings for $country{name}", (@dates * 6) ) if not $opt_quiet; - -# Grab requested data -# [ [0], $_->[1], $_->[2])) { -## $writer->write_programme($_); -## } -## $mainbar->update() if not $opt_quiet; -##} - -# Data store before being written as XML -my @programmes = (); - -# Fetch the data -# (note the params are all globals so the params aren't strictly necessary -# but let's try for some better programming practice ;-) -get_channel_programs(\@dates, \@channels, \@ch_all, \@programmes); - -# Write the elements -foreach (@programmes) { - $writer->write_programme($_); -} - -# end the progress bar -$mainbar->finish() if not $opt_quiet; - -# close xml file -$writer->end(); - -# Signal that something went wrong if there were warnings. -exit(1) if $warnings; - -# All data fetched ok. -#debug "Exiting without warnings."; -exit(0); diff -Nru xmltv-0.6.1/grab/es_laguiatv/test.conf xmltv-0.6.3/grab/es_laguiatv/test.conf --- xmltv-0.6.1/grab/es_laguiatv/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/es_laguiatv/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,179 +0,0 @@ -configversion 3 -usecache no -cachedir /tmp/.xmltv/cache -getdescriptions yes -#channel 13tv-943 13TV -#channel 40-latino-759 40 Latino -#channel 40tv-866 40TV -#channel 7rm-935 7RM -#channel axn-867 AXN -#channel axn-hd-868 AXN HD -#channel al-jazeera-english-772 Al Jazeera English -#channel andalucia-tv-783 Andalucía TV -#channel animax-779 Animax -channel antena-3-839 Antena 3 -#channel aragon-tv-919 Aragón TV -#channel bbc-world-776 BBC World -#channel baby-tv-790 Baby TV -#channel barcatv-764 BarçaTV -#channel biography-771 Biography -#channel bloomberg-774 Bloomberg -#channel boing-869 Boing -#channel cmt-1-929 CMT 1 -#channel cmt-2-931 CMT 2 -#channel cnbc-778 CNBC -#channel cnn-int-773 CNN Int -#channel calle-13-870 Calle 13 -#channel canal-865 Canal + -#channel canal-2-871 Canal + 2 -#channel canal-3d-803 Canal + 3D -#channel canal-accion-710 Canal + Acción -#channel canal-accion-30-872 Canal + Acción 30 -#channel canal-accion-hd-908 Canal + Acción HD -#channel canal-comedia-711 Canal + Comedia -#channel canal-comedia-hd-909 Canal + Comedia HD -#channel canal-dcine-712 Canal + DCine -#channel canal-dcine-hd-873 Canal + DCine HD -#channel canal-deportes-715 Canal + Deportes -#channel canal-deportes-hd-791 Canal + Deportes HD -#channel canal-eventos-716 Canal + Eventos -#channel canal-futbol-713 Canal + Fútbol -#channel canal-futbol-hd-714 Canal + Fútbol HD -#channel canal-hd-874 Canal + HD -#channel canal-liga-hd-875 Canal + Liga HD -#channel canal-xtra-877 Canal + XTRA -#channel canal-xtra-hd-718 Canal + Xtra HD -#channel canal-33-951 Canal 33 -#channel canal-club-787 Canal Club -#channel canal-cocina-767 Canal Cocina -#channel canal-extremadura-939 Canal Extremadura -#channel canal-fox-731 Canal Fox -#channel canal-sur-863 Canal Sur -#channel canal-sur-2-864 Canal Sur 2 -#channel canal-de-las-estrellas-786 Canal de las Estrellas -#channel canal-1-30-876 Canal+ 1...30 -#channel canal-deportes-2-hd-755 Canal+ Deportes 2 HD -#channel canal-deportes-mixto-910 Canal+ Deportes Mixto -#channel canal-liga-2-792 Canal+ Liga 2 -#channel canal-liga-2-a-735 Canal+ Liga 2 A -#channel canal-liga-2-b-736 Canal+ Liga 2 B -#channel canal-liga-2-c-737 Canal+ Liga 2 C -#channel canal-liga-2-d-738 Canal+ Liga 2 D -#channel canal-liga-2-e-739 Canal+ Liga 2 E -#channel cartoon-network-878 Cartoon Network -#channel cartoonito-793 Cartoonito -#channel caza-y-pesca-769 Caza y Pesca -#channel clan-756 Clan -#channel cosmopolitan-879 Cosmopolitan -#channel cosmopolitan-hd-717 Cosmopolitan HD -#channel crimen-investigacion-989 Crimen & Investigación -channel cuatro-895 Cuatro -#channel cubavision-805 Cubavisión -#channel d-cine-espanol-730 D Cine Español -#channel decasa-770 Decasa -#channel discovery-channel-766 Discovery Channel -#channel discovery-max-995 Discovery MAX -#channel disney-channel-880 Disney Channel -#channel disney-channel-hd-965 Disney Channel HD -#channel disney-cinemagic-881 Disney Cinemagic -#channel disney-cinemagic-hd-719 Disney Cinemagic HD -#channel disney-junior-983 Disney Junior -#channel disney-xd-794 Disney XD -#channel divinity-925 Divinity -#channel ehs-788 EHS -#channel etb-1-855 ETB 1 -#channel etb-2-856 ETB 2 -#channel etb-3-927 ETB 3 -#channel etb-sat-975 ETB Sat -#channel energy-963 Energy -#channel eurosport-882 Eurosport -#channel eurosport-hd-751 Eurosport HD -#channel fdf-734 FDF -#channel france-24-775 FRANCE 24 -#channel fox-crime-732 Fox Crime -#channel fox-hd-883 Fox HD -#channel fox-news-777 Fox News -#channel galicia-tv-977 Galicia TV -#channel golf-761 Golf+ -#channel historia-768 Historia -#channel hogar-util-912 Hogar Útil -#channel hollywood-728 Hollywood -#channel hollywood-hd-795 Hollywood HD -#channel ib3-937 IB3 -#channel infometeo-729 InfoMeteo -#channel intereconomia-959 Intereconomía -#channel la-10-923 La 10 -#channel la-2-823 La 2 -#channel la-sexta-782 La Sexta -#channel la-sexta-2-887 La Sexta 2 -#channel la-sexta-3-888 La Sexta 3 -#channel la-siete-720 La Siete -#channel mtv-2-802 MTV 2 -#channel mtv-esp-740 MTV ESP -#channel mtv-hd-752 MTV HD -#channel marca-tv-884 Marca TV -#channel mezzo-760 Mezzo -#channel mezzo-live-hd-721 Mezzo Live HD -#channel multi-x-1-796 Multi-X (1) -#channel multi-x-2-797 Multi-X (2) -#channel multi-x-3-798 Multi-X (3) -#channel multideporte-1-742 Multideporte 1 -#channel multideporte-2-743 Multideporte 2 -#channel multideporte-3-744 Multideporte 3 -#channel multideporte-4-745 Multideporte 4 -#channel multideporte-5-746 Multideporte 5 -#channel multideporte-6-747 Multideporte 6 -#channel multideporte-7-748 Multideporte 7 -#channel nhk-world-799 NHK World -#channel nick-jr-741 NICK JR -#channel nat-geo-wild-hd-913 Nat Geo Wild HD -#channel nat-geographic-hd-753 Nat Geographic HD -#channel nat-geographic-wild-904 Nat Geographic Wild -#channel national-geographic-886 National Geographic -#channel neox-722 Neox -#channel nickelodeon-905 Nickelodeon -#channel nitro-800 Nitro -#channel nou-860 Nou -#channel nou-2-861 Nou 2 -#channel nova-723 Nova -#channel nueve-993 Nueve -#channel odisea-765 Odisea -#channel panda-754 Panda -#channel paramount-channel-979 Paramount Channel -#channel paramount-comedy-733 Paramount Comedy -#channel playboy-tv-757 Playboy TV -#channel playhouse-disney-758 Playhouse Disney -#channel rt-espanol-801 RT Español -#channel radios-789 Radios. -#channel real-madrid-tv-763 Real Madrid TV -#channel russia-today-749 Russia Today -#channel syfy-750 SYFY -#channel syfy-hd-906 SYFY HD -#channel sony-entertaiment-tv-780 Sony Entertaiment TV -#channel sportmania-889 Sportmanía -#channel super-3-859 Super 3 -#channel tcm-890 TCM -#channel tcm-autor-971 TCM Autor -#channel tcm-clasico-726 TCM Clásico -#channel tl-novelas-europa-973 TL Novelas Europa -#channel tnt-727 TNT -#channel tpa-941 TPA -#channel tv-canaria-933 TV Canaria -#channel tv-record-806 TV RECORD -#channel tv3-858 TV3 -#channel tv3cat-785 TV3Cat -#channel tv5-891 TV5 -channel tve-1-807 TVE 1 -#channel tvg-862 TVG -#channel tvod-915 TVOD -#channel taquilla-hd-804 Taquilla HD -channel tele-5-847 Tele 5 -#channel teledeporte-762 Teledeporte -#channel telemadrid-857 Telemadrid -#channel telemadrid-sat-laotra-784 Telemadrid SAT/LaOtra -#channel telesur-725 Telesur -#channel vh1-892 VH1 -#channel viajar-893 Viajar -#channel viajar-hd-724 Viajar HD -#channel xplora-981 Xplora -#channel laotra-953 laOtra diff -Nru xmltv-0.6.1/grab/es_laguiatv/tv_grab_es_laguiatv xmltv-0.6.3/grab/es_laguiatv/tv_grab_es_laguiatv --- xmltv-0.6.1/grab/es_laguiatv/tv_grab_es_laguiatv 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/es_laguiatv/tv_grab_es_laguiatv 1970-01-01 00:00:00.000000000 +0000 @@ -1,1091 +0,0 @@ -#!/usr/bin/perl -w - -=pod - -=head1 NAME - -tv_grab_es_laguiatv - Alternative TV grabber for Spain. - -=head1 SYNOPSIS - -tv_grab_es_laguiatv --help - -tv_grab_es_laguiatv [--config-file FILE] --configure [--gui OPTION] - -tv_grab_es_laguiatv [--config-file FILE] [--output FILE] [--days N] - [--offset N] [--quiet] - -tv_grab_es_laguiatv --list-channels - -tv_grab_es_laguiatv --capabilities - -tv_grab_es_laguiatv --version - -=head1 DESCRIPTION - -Output TV listings for spanish channels from www.laguiatv.com. -Supports analogue and digital (D+) channels. -The grabber relies on parsing HTML so it might stop working at any time. - -First run B to choose, which channels you want -to download. Then running B with no arguments will output -listings in XML format to standard output. - -B<--configure> Prompt for which channels, -and write the configuration file. - -B<--config-file FILE> Set the name of the configuration file, the -default is B<~/.xmltv/tv_grab_es_laguiatv.conf>. This is the file written by -B<--configure> and read when grabbing. - -B<--gui OPTION> Use this option to enable a graphical interface to be used. -OPTION may be 'Tk', or left blank for the best available choice. -Additional allowed values of OPTION are 'Term' for normal terminal output -(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar. - -B<--output FILE> Write to FILE rather than standard output. - -B<--days N> Grab N days. The default is 3. - -B<--offset N> Start N days in the future. The default is to start -from today. - -B<--quiet> Suppress the progress messages normally written to standard -error. - -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L - -B<--version> Show the version of the grabber. - -B<--help> Print a help message and exit. - -=head1 SEE ALSO - -L. - -=head1 AUTHOR - -CandU, candu_sf@sourceforge.net, based on tv_grab_es, from Ramon Roca. - -=head1 BUGS - -=cut - -# - - -###################################################################### -# initializations - -use strict; -use XMLTV; -use XMLTV::Version "$XMLTV::VERSION"; -use XMLTV::Capabilities qw/baseline manualconfig cache/; -use XMLTV::Description 'Spain (laguiatv.com)'; -use Getopt::Long; -use Date::Manip; -use HTML::TreeBuilder; -use HTML::Entities; # parse entities -use IO::File; -use DateTime; - -use LWP::Simple; -use Encode; - -use XMLTV::Memoize; -use XMLTV::ProgressBar; -use XMLTV::Ask; -use XMLTV::Config_file; -use XMLTV::DST; -use XMLTV::Get_nice 0.005065; -use XMLTV::Mode; -use XMLTV::Date; -# Todo: perhaps we should internationalize messages and docs? -use XMLTV::Usage < 'http://www.laguiatv.com/programacion/', - 'source-data-url' => "http://www.laguiatv.com/programacion/", - 'generator-info-name' => 'XMLTV', - 'generator-info-url' => 'http://xmltv.org/', - }; - -my $WRITE_ZERO_LENGTH = 0; # whether zero-length programmes should be included in the output. -my $DO_SLOWER_DESC_GET = 0; -my $CONFIG_VERSION = 1; # default to v1 (v1 doesnt have version info) -my $EXPECTED_CONFIG_VERSION = 3; -my $CONFIG_USECACHE = 0; # whether to use a disc cache for web pages -my $CONFIG_CACHEDIR; # directory to store cached web pages - -# default language -my $LANG="es"; - -# default web page encoding -my $WEB_ENCODING = 'iso-8859-15'; - -# Global channel_data -our @ch_all; - -my @hide_channels = ( - "canal-bar.a", # currently gives 404 not found -); - - -###################################################################### -# get options - -# Get options, including undocumented --cache option. -XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux'); -my ($opt_days, $opt_offset, $opt_help, $opt_output, - $opt_configure, $opt_config_file, $opt_gui, - $opt_quiet, $opt_list_channels, $opt_debug); -$opt_days = 4; # default -$opt_offset = 0; # default -$opt_quiet = 0; # default -$opt_debug = 0; # default -GetOptions('days=i' => \$opt_days, - 'offset=i' => \$opt_offset, - 'help' => \$opt_help, - 'configure' => \$opt_configure, - 'config-file=s' => \$opt_config_file, - 'gui:s' => \$opt_gui, - 'output=s' => \$opt_output, - 'quiet' => \$opt_quiet, - 'list-channels' => \$opt_list_channels, - 'debug' => \$opt_debug, - ) - or usage(0); - -# Force days to be 1, since we get all days at once -# $opt_days = 1; -die 'number of days must not be negative' - if (defined $opt_days && $opt_days < 0); -usage(1) if $opt_help; - -# [mod Jan 2014 - max days is 4 -die 'max days available is 4 (today + 3)' - if ( $opt_offset + $opt_days > 4 ); - -XMLTV::Ask::init($opt_gui); - - -# Although we use HTTP::Cache::Transparent, this undocumented --cache -# option for debugging is still useful since it will _always_ use a -# cached copy of a page, without contacting the server at all. -# -use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux'); - - -# debug print function -sub debug_print -{ - print STDERR $_[0]."\n" if $opt_debug; -} - -my $mode = XMLTV::Mode::mode('grab', # default - $opt_configure => 'configure', - $opt_list_channels => 'list-channels', - ); - -# File that stores which channels to download. -my $config_file - = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_es_laguiatv', $opt_quiet); - -my @config_lines; # used only in grab mode -if ($mode eq 'configure') { - XMLTV::Config_file::check_no_overwrite($config_file); -} -elsif ($mode eq 'grab') { - @config_lines = XMLTV::Config_file::read_lines($config_file); -} -elsif ($mode eq 'list-channels') { - # Config file not used. -} -else { die } - -# Whatever we are doing, we need the channels data. -my %channels = get_channels(); # sets @ch_all -my @channels; - -my %icons; - -my %categories = ( - "tag-a" => "Cine", - "tag-b" => "Deportes", - "tag-c" => "Programas", - "tag-d" => "Series", - "tag-e" => "Noticias" -); - -###################################################################### -# write configuration - -if ($mode eq 'configure') { - %channels = get_channels(); - - open(CONF, ">$config_file") or die "cannot write to $config_file: $!"; - - print CONF "configversion 3\n"; - - # Ask about using a cache - my $usecache = ask_boolean("Do you want to use a cache for web pages (recommended)", 'yes'); - warn("cannot read input, using default") - if not defined $usecache; - - print CONF "usecache "; - print CONF "yes\n" if $usecache; - print CONF "no\n" if not $usecache; - - my $cachedir = "$ENV{HOME}/.xmltv/cache"; - if ($usecache) - { - my $cachedir = ask("Directory for cache (default=$cachedir)"); - warn("cannot read input, using default") - if not defined $cachedir; - } - print CONF "cachedir ".$cachedir."\n"; - - # Ask about getting descs - my $getdescs = ask_boolean("Do you want to get descriptions (very slow)", 'yes'); - warn("cannot read input, using default") - if not defined $getdescs; - - print CONF "getdescriptions "; - print CONF "yes\n" if $getdescs; - print CONF "no\n" if not $getdescs; - - #my $cacheicons = ask_boolean('Do you want to get and cache icons during configure', 'yes'); - #warn("cannot read input, using default") - # if not defined $cacheicons; - - # Ask about each channel. - my @chs = sort { $channels{$a} cmp $channels{$b} } keys %channels; - my @names = map { $channels{$_} } @chs; - my @qs = map { "Add channel $_?" } @names; - my @want = ask_many_boolean(1, @qs); - - #my $iconbar = new XMLTV::ProgressBar({name => 'getting icon urls', count => scalar @chs}) - #if ((not $opt_quiet) && $cacheicons); - - foreach (@chs) { - my $w = shift @want; - warn("cannot read input, stopping channel questions"), last - if not defined $w; - # No need to print to user - XMLTV::Ask is verbose enough. - - # Print a config line, but comment it out if channel not wanted. - print CONF '#' if not $w; - my $name = shift @names; -# if ($cacheicons) -# { -# my $icon = get_icon($_); -# print CONF "channel $_ $name icon:$icon\n"; -# } -# else -# { - print CONF "channel $_ ".encode($WEB_ENCODING, $name)."\n"; -# } - # TODO don't store display-name in config file. - -# update $iconbar if ((not $opt_quiet) && $cacheicons); - } - - close CONF or warn "cannot close $config_file: $!"; - say("Finished configuration."); - - exit(); -} - - -# Not configuration, we must be writing something, either full -# listings or just channels. -# -die if $mode ne 'grab' and $mode ne 'list-channels'; - -# Options to be used for XMLTV::Writer. -my %w_args; -if (defined $opt_output) { - my $fh = new IO::File(">$opt_output"); - die "cannot write to $opt_output: $!" if not defined $fh; - $w_args{OUTPUT} = $fh; -} -$w_args{encoding} = 'UTF-8'; -my $writer = new XMLTV::Writer(%w_args); -$writer->start($HEAD); - -if ($mode eq 'list-channels') { - $writer->write_channel($_) foreach @ch_all; - $writer->end(); - exit(); -} - -###################################################################### -# We are producing full listings. -die if $mode ne 'grab'; - -# Read configuration -my $line_num = 1; -foreach (@config_lines) { - ++ $line_num; - next if not defined; - - if (/configversion:?\s+(\S+)/) - { - $CONFIG_VERSION = $1; - } - elsif (/usecache:?\s+(\S+)/) - { - if($1 eq "yes") - { - $CONFIG_USECACHE = 1; - } - } - elsif (/cachedir:?\s+(\S+)/) - { - $CONFIG_CACHEDIR = $1; - } - elsif (/getdescriptions:?\s+(\S+)/) - { - if("$CONFIG_VERSION" ne "$EXPECTED_CONFIG_VERSION") - { - die "Config file is out of date, please rerun with --configure\n"; - } - if($1 eq "yes") - { - $DO_SLOWER_DESC_GET = 1; - } - } - elsif (/^channel:?\s+(\S+)\s+([^#]+)icon\:([^#]+)/) - { - my $ch_did = $1; - my $ch_name = $2; - my $ch_icon = $3; - - - #debug_print "Got channel $ch_name icon $ch_icon\n"; - $ch_name =~ s/\s*$//; - push @channels, $ch_did; - $channels{$ch_did} = $ch_name; - $icons{$ch_did} = $ch_icon; - } - elsif (/^channel:?\s+(\S+)\s+([^#]+)/) - { - my $ch_did = $1; - my $ch_name = $2; - - debug_print "Fetching channel $ch_name"; - $ch_name =~ s/\s*$//; - push @channels, $ch_did; - $channels{$ch_did} = $ch_name; - } - else { - warn "$config_file:$line_num: bad line\n"; - } -} - - - -if ($CONFIG_USECACHE) { -use HTTP::Cache::Transparent; -HTTP::Cache::Transparent::init( { - BasePath => $CONFIG_CACHEDIR, - NoUpdate => 60*60, # cache time in seconds - MaxAge => 4, # flush time in hours - Verbose => $opt_debug, -} ); -} - - - - -###################################################################### -# begin main program - -# Assume the listings source uses CET (see BUGS above). -my $now = DateCalc(parse_date('now'), "$opt_offset days"); -die "No channels specified, run me with --configure\n" - if not keys %channels; -my @to_get; - -my $iconbar = new XMLTV::ProgressBar({name => 'getting channel info', count => scalar @channels}) - if not $opt_quiet; -# the order in which we fetch the channels matters -foreach my $ch_did (@channels) { - my $ch_name=$channels{$ch_did}; - my $ch_xid="$ch_did.laguiatv.com"; -# my $ch_icon=$icons{$ch_did}; -# if (!$ch_icon) -# { -# $ch_icon = get_icon($ch_did); -# } -# -# if(index($ch_icon, "shim.gif") < 0) -# { -# $writer->write_channel({ id => $ch_xid, -# 'display-name' => [ [ $ch_name ] ] , -# 'icon' => [ { 'src' => $ch_icon } ] }); -# } -# else -# { - $writer->write_channel({ id => $ch_xid, - 'display-name' => [ [ $ch_name ] ] }); -# } - - # [Jan 2014] - current website offers a fixed 4 days of data - # my $day=UnixDate($now,'%Q'); - # for (my $i=0;$i<$opt_days;$i++) { - # push @to_get, [ $day, $ch_xid, $ch_did ]; - # #for each day - # $day=nextday($day); die if not defined $day; - # } - # - push @to_get, [ '', $ch_xid, $ch_did ]; - - update $iconbar if not $opt_quiet; -} - -# This progress bar is for both downloading and parsing. Maybe -# they could be separate. -# -my $bar = new XMLTV::ProgressBar({name => 'getting listings', count => scalar @to_get}) - if not $opt_quiet; -foreach (@to_get) { - debug_print "process $_->[0], $_->[1], $_->[2]\n"; - foreach (process_table($_->[0], $_->[1], $_->[2])) { - $writer->write_programme($_); - } - update $bar if not $opt_quiet; -} -$bar->finish() if not $opt_quiet; -$writer->end(); - -###################################################################### -# subroutine definitions - -# Use Log::TraceMessages if installed. -BEGIN { - eval { require Log::TraceMessages }; - if ($@) { - *t = sub {}; - *d = sub { '' }; - } - else { - *t = \&Log::TraceMessages::t; - *d = \&Log::TraceMessages::d; - Log::TraceMessages::check_argv(); - } -} - -#### -# process_table: fetch a URL and process it -# -# arguments: -# Date::Manip object giving the day to grab -# xmltv id of channel -# elpais.es id of channel -# -# returns: list of the programme hashes to write -# -sub process_table { - - my ($date, $ch_xmltv_id, $ch_es_id) = @_; - - my $ch_conv_id = convert_id_to_laguiatvid($ch_es_id); - my $today = UnixDate($date, '%d/%m/%Y'); - - my $url = 'http://www.laguiatv.com/programacion/'.$ch_es_id.'.html'; - debug_print "Getting $url\n"; - t $url; - local $SIG{__WARN__} = sub - { - warn "$url: $_[0]"; - }; - - # parse the page to a document object - my $tree; - # HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages! - # Save the page and run it again and you don't get the warning! - # You can't even supress the warning! What a crock. - { - local $SIG{__WARN__} = sub { - warn @_ unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/); - }; - $tree = get_nice_tree($url,'',$WEB_ENCODING); - } - - my @program_data = get_program_data($tree); - my $bump_start_day=0; - - my @r; - while (@program_data) { - my $cur = shift @program_data; - my $next = shift @program_data; - unshift @program_data,$next if $next; - - my $p = make_programme_hash($date, $ch_xmltv_id, $ch_es_id, $cur, $next); - if (not $p) { - require Data::Dumper; - my $d = Data::Dumper::Dumper($cur); - warn "cannot write programme on $ch_xmltv_id on $date:\n$d\n"; - } - else { - push @r, $p; - } - -# if (!$bump_start_day && bump_start_day($cur,$next)) { -# #$bump_start_day=1; -# $date = UnixDate(DateCalc($date,"+ 1 day"),'%Q'); -# } - } - return @r; -} - -sub make_programme_hash { - my ($date, $ch_xmltv_id, $ch_es_id, $cur, $next) = @_; - - #require Data::Dumper; debug_print Data::Dumper::Dumper($cur); - - my %prog; - - $prog{channel}=$ch_xmltv_id; - $prog{title}=[ [ encode( 'UTF-8', $cur->{title} ), $LANG ] ]; - $prog{"sub-title"}=[ [ encode( 'UTF-8', $cur->{subtitle} ), $LANG ] ] if defined $cur->{subtitle}; - # $prog{category}=[ [ $cur->{category}, $LANG ] ]; - $prog{start}=$cur->{stime}; - $prog{stop} =$cur->{etime} if defined $cur->{etime}; - $prog{desc}=[ [ encode( 'UTF-8', $cur->{desc} ), $LANG ] ] if defined $cur->{desc}; - # $prog{category}=[ [ encode( 'UTF-8', $cur->{category} ), $LANG ] ] if defined $cur->{category}; - $prog{'date'} = $cur->{year} if defined $cur->{year}; - $prog{'star-rating'} = [ $cur->{rating} . '/5' ] if defined $cur->{rating}; - $prog{'rating'} = [[ $cur->{classification}, '' ]] if defined $cur->{classification}; - - if (defined $cur->{genres}) - { - foreach ( @{ $cur->{genres} } ) - { - push @{$prog{'category'}}, [ encode('UTF-8', $_), $LANG ] if $_ ne ''; - } - } - if (defined $cur->{directors}) - { - foreach ( @{ $cur->{directors} } ) - { - push @{$prog{'credits'}{'director'}}, encode('UTF-8', $_) if $_ ne ''; - } - } - if (defined $cur->{actors}) - { - foreach ( @{ $cur->{actors} } ) - { - push @{$prog{'credits'}{'actor'}}, encode('UTF-8', $_) if $_ ne ''; - } - } - - - return \%prog; -} -sub bump_start_day { - my ($cur,$next) = @_; - if (!defined($next)) { - return undef; - } - my $start = UnixDate($cur->{stime},'%H:%M'); - my $stop = UnixDate($next->{stime},'%H:%M'); - if (Date_Cmp($start,$stop)>0) { - return 1; - } else { - return 0; - } -} - - -# -sub get_program_data -{ - my ($tree) = @_; - my @data; - - my $today = DateTime->today->set_time_zone('Europe/Madrid'); - - # - current website offers a fixed 4 days of data - # ignore any programmes outside requested range - my $startgrab = $today->clone->add('days' => $opt_offset)->epoch(); - my $stopgrab = $today->clone->add('days' => ($opt_offset + $opt_days))->epoch(); - debug_print 'Grab times: start: '.DateTime->from_epoch(epoch=>$startgrab)->strftime("%Y %m %d %H%M %S %z").' stop: '.DateTime->from_epoch(epoch=>$stopgrab)->strftime("%Y %m %d %H%M %S %z"); - # find schedule table - - # the following could could do with some error checking but I don't have time to do that right now :-( - - my @divs = $tree->look_down('_tag' => 'div', 'id' => qr/dia1|nad2|nad3|nad4/); - - foreach my $div (@divs) - { - my ($i) = $div->attr('id') =~ /(?:dia|nad)(\d)/; - #'debugtime' debug_print "i= $i ".$div->attr('id'); - #'debugtime' debug_print 'today: '.$today->strftime("%Y %m %d %H%M %S %z"); - my $theday = $today->clone->add(days => ($i - 1)); - #'debugtime' debug_print 'theday: '.$theday->strftime("%Y %m %d %H%M %S %z"); - - my @trs = $div->look_down('_tag' => 'tr'); - - foreach my $tr (@trs) - { - - my $stime = $tr->look_down('_tag' => 'th')->as_text; - trim($stime); - - my $p_div = $tr->look_down('_tag' => 'div', 'class' => 'programa'); - next if !$p_div; - - my $a = $p_div->look_down('_tag' => 'a'); - - my $p_url = $a->attr('href'); - my $p_title = $a->as_text; - - my $p_times = $p_div->look_down('_tag' => 'p')->as_text; - - my ($h, $i, $h2, $i2) = $p_times =~ /(\d*):(\d*)(?: *a *(\d*):(\d*))?/; - - my $showtime = $theday->clone->set(hour => $h, minute => $i, second => 0); - - # - current website offers a fixed 4 days of data - # ignore any programmes outside requested range - #'debugtime' debug_print 'this: '.$showtime->strftime("%Y %m %d %H%M %S %z"); - next if ( $showtime->epoch() < $startgrab ) || ( $showtime->epoch() >= $stopgrab ); - - my $p_stime = $theday->clone->set(hour => $h, minute => $i, second => 0)->strftime("%Y%m%d%H%M%S %z"); - - my $p_etime; - # this will probably fail around DST times - if (defined $h2 && $h2 >= 0) - { - $showtime->add(days => 1) if $h2 < $h; - eval { # try - $showtime->set(hour => $h2, minute => $i2, second => 0); - $p_etime = $showtime->strftime("%Y%m%d%H%M%S %z"); - } or do { # catch - # no output prog 'stop' time - } - } - - - # get descriptions? Kinda compulsory now since there is no longer *any* description on the schedule page - # - my ($p_description, $p_rating, $p_classification, $p_year, @p_genres, @p_actors, @p_directors) = ('', '', '', '', (), (), ()); - - # - { # begin code block - if ($DO_SLOWER_DESC_GET) # get descriptions - { - - my $url = $p_url; - debug_print "Getting $url"; - t $url; - - last if $url eq 'javascript:void(0);' ; - - # handle no programme info situation (probably means "Close"?) : - # - # 04:00 - # - #
    - #

    *

    - #

    04:00 a 06:00

    - #
    - # - # - last if $url =~ m%programas/-\d*/$%; - - # parse the page to a document object - # HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages! - # Often on the http://hoycinema.abc.es/ pages. (Could be due to the - # - my($match) = ($text =~ /window.__INITIAL_STATE__ = (\{.+\});/); - - if ($match) { - my $decoded = JSON->new->decode($match); - - if (ref($decoded) eq "HASH") { - my $data = $decoded; - - #debug(5, JSON->new->pretty->encode($decoded)); - - # step through hashes using key sequence - foreach my $key (@{$keys}) { - debug(5, "Looking for JSON key $key"); - return unless exists $data->{$key}; - $data = $data->{$key}; - } - debug(5, "Found JSON data"); + my $decoded = JSON->new->decode($text); - #debug(5, JSON->new->pretty->encode($data)); - #debug(5, "KEYS: ", join(", ", sort keys %{$data})); - return($data); - } + if (ref($decoded) eq "HASH") { + # debug(5, JSON->new->pretty->encode($decoded)); + return $decoded->{response}; } } return; } +# cache for group name to API ID mapping +my %group2id; + # Grab channel list sub channels { # Fetch & extract JSON sub-part - my $data = _getJSON("tanaan", "peruskanavat", - ["channelGroups", - "channelGroupsArray"]); + my $data = _getJSON(""); # - # Channels data has the following structure + # channel-groups response has the following structure # # [ # { + # id => "default_builtin_channelgroup1" # slug => "peruskanavat", # channels => [ # { @@ -103,15 +86,19 @@ foreach my $item (@{$data}) { if ((ref($item) eq "HASH") && + (exists $item->{id}) && (exists $item->{slug}) && (exists $item->{channels}) && (ref($item->{channels}) eq "ARRAY")) { - my $group = $item->{slug}; - my $channels = $item->{channels}; + my($api_id, $group, $channels) = @{$item}{qw(id slug channels)}; - if (defined($group) && length($group) && + if (defined($api_id) && length($api_id) && + defined($group) && length($group) && (ref($channels) eq "ARRAY")) { - debug(2, "Source telkku.com found group '$group' with " . scalar(@{$channels}) . " channels"); + debug(2, "Source telkku.com found group '$group' ($api_id) with " . scalar(@{$channels}) . " channels"); + + # initialize group name to API ID map + $group2id{$group} = $api_id; foreach my $channel (@{$channels}) { if (ref($channel) eq "HASH") { @@ -140,6 +127,15 @@ return; } +sub _group2id($) { + my($group) = @_; + + # Make sure group to ID map is initialized + channels() unless %group2id; + + return $group2id{$group}; +} + # Grab one day sub grab { my($self, $id, $yesterday, $today, $tomorrow, $offset) = @_; @@ -147,17 +143,26 @@ # Get channel number from XMLTV id return unless my($channel, $group) = ($id =~ /^([\w-]+)\.([\w-]+)\.telkku\.com$/); - # Fetch & extract JSON sub-part - my $data = _getJSON($today, $group, - ["offeringByChannelGroup", - $group, - "offering", - "publicationsByChannel"]); + # Map group name to API ID + return unless my $api_id = _group2id($group); + + # + # API parameters: + # + # - date is $today + # - range is 24 hours (start 00:00:00.000 - end 00:00:00.000) + # - max. 1000 entries per channel + # - detailed information + # + # Response will include programmes from $yesterday that end $today, to + # $tomorrow where a programme of $today ends. + # + my $data = _getJSON("$api_id/offering?endTime=00:00:00.000&limit=1000&startTime=00:00:00.000&view=PublicationDetails&tvDate=" . $today->ymdd()); # # Programme data has the following structure # - # [ + # publicationsByChannel => [ # { # channel => { # id => "yle-tv1", @@ -178,10 +183,11 @@ # ... # ] # - if (ref($data) eq "ARRAY") { + if ((ref($data) eq "HASH") && + (ref($data->{publicationsByChannel}) eq "ARRAY")) { my @objects; - foreach my $item (@{$data}) { + foreach my $item (@{ $data->{publicationsByChannel} }) { if ((ref($item) eq "HASH") && (ref($item->{channel}) eq "HASH") && (ref($item->{publications}) eq "ARRAY") && diff -Nru xmltv-0.6.1/grab/fi/fi/source/telsu.pm xmltv-0.6.3/grab/fi/fi/source/telsu.pm --- xmltv-0.6.1/grab/fi/fi/source/telsu.pm 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fi/fi/source/telsu.pm 2020-09-07 15:02:53.000000000 +0000 @@ -116,7 +116,7 @@ if ($title && $time && $desc) { if (my($new, $start_h, $start_m, $end_h, $end_m) = - $time->as_text() =~ /^(.+)\s(\d{2}):(\d{2})\s-\s(\d{2}):(\d{2})/) { + $time->as_text() =~ /^(.+)\s(\d{2})[:.](\d{2})\s-\s(\d{2})[:.](\d{2})/) { $title = $title->as_text(); $desc = $desc->as_text(); diff -Nru xmltv-0.6.1/grab/fi/fi/source/yle.pm xmltv-0.6.3/grab/fi/fi/source/yle.pm --- xmltv-0.6.1/grab/fi/fi/source/yle.pm 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fi/fi/source/yle.pm 2020-09-07 15:02:53.000000000 +0000 @@ -1,6 +1,6 @@ # -*- mode: perl; coding: utf-8 -*- ########################################### # -# tv_grab_fi: source specific grabber code for http://www.yle.fi +# tv_grab_fi: source specific grabber code for https://www.yle.fi # ############################################################################### # @@ -122,7 +122,7 @@ # if (my $div = $root->look_down("_tag" => "div", "aria-label" => qr/^${channel}$/)) { - if (my $parent = $div->look_up("class" => "guide-channels__channel")) { + if (my $parent = $div->look_up("class" => qr/guide-channels__channel/)) { if (my @programmes = $parent->look_down("class" => qr/^schedule-card\s+/)) { foreach my $programme (@programmes) { my $start = $programme->look_down("itemprop", "startDate"); diff -Nru xmltv-0.6.1/grab/fi/test.conf xmltv-0.6.3/grab/fi/test.conf --- xmltv-0.6.1/grab/fi/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fi/test.conf 2020-09-07 15:02:53.000000000 +0000 @@ -68,6 +68,7 @@ #channel 7.iltapulu.fi Liv #channel 8.iltapulu.fi FOX #channel 9.iltapulu.fi Yle Teema & Fem +##channel alfatv.ampparit.com AlfaTV ##channel AlfaTV.fi.yle.fi AlfaTV ##channel alfatv.muut.telkku.com AlfaTV ##channel AlfaTV.sv.yle.fi AlfaTV @@ -92,7 +93,6 @@ ##channel cmore-first.elokuvat.telkku.com C More First ##channel cmore-first-hd.elokuvat.telkku.com C More First HD ##channel cmore_first.telsu.fi C More First -##channel c-more-golf.ampparit.com C More Golf HD ##channel cmore_golfhd.telsu.fi C More Golf HD ##channel cmore-golf-hd.urheilu.telkku.com C More Golf HD ##channel c-more-hits.ampparit.com C More Hits @@ -223,6 +223,7 @@ ##channel nick-jr.lapset.telkku.com Nick Jr ##channel nickjr.telsu.fi Nick Jr ##channel outdoor-channel.lifestyle.telkku.com Outdoor Channel +##channel paramount-network.muut.telkku.com Paramount Network ##channel playboy-tv.lifestyle.telkku.com Playboy TV ##channel rtl2.telsu.fi RTL II ##channel rtl.muut.telkku.com RTL diff -Nru xmltv-0.6.1/grab/fr/test.conf xmltv-0.6.3/grab/fr/test.conf --- xmltv-0.6.1/grab/fr/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fr/test.conf 2020-09-07 15:02:53.000000000 +0000 @@ -1,167 +1,27 @@ -# channel 13emerue -# channel 6ter -# channel ab1 -# channel abmoteurs -# channel action -# channel aljazeera -# channel animaux -# channel arte -# channel artebe -# channel astrocentertv -# channel babytv -# channel bbcworld -# channel be1 -# channel becine -# channel beinsports1 -# channel beinsports2 -# channel beseries -# channel besport1 -# channel besport2 -# channel bestofshopping -# channel bfmbusiness -# channel bfmtv -# channel bloombergtv -# channel boing -# channel boomerang -# channel bravahdtv -# channel brazzerstv -# channel canalj -# channel canalplus -# channel canalpluscinema -# channel canalplusdecale -# channel canalplusfamily -# channel canalplusseries -# channel canalplussport -# channel cartoonnetwork -# channel cctvf -# channel chasseetpeche -# channel cinefx -# channel cineplusclassic -# channel cineplusclassicbe -# channel cineplusclub -# channel cineplusemotion -# channel cineplusfamiz -# channel cineplusfrisson -# channel cineplusfrissonbe -# channel cinepluspremier -# channel cinepluspremierbe -# channel cinepolar -# channel clubrtl -# channel cmusictv -# channel cnbc -# channel cnninternational -# channel comedieplus -# channel d17 -# channel d8 -# channel discoverychannel -# channel discoveryscience -# channel disneychannel -# channel disneychannelplus1 -# channel disneyjunior -# channel disneyxd -# channel eentertainment -# channel equidialife -# channel equidialive -# channel eurosport -# channel eurosport2 -# channel extremesportschannel -channel france2 -# channel france24 -# channel france3 -# channel france4 -# channel france5 -# channel franceo -# channel gameone -# channel girondinstv -# channel golfplus -# channel grandlilletv -# channel gulli -# channel hd1 -# channel histoire -# channel infosport -# channel itele -# channel jone -# channel june -# channel just4talent -# channel karaokechannel -# channel kto -# channel ladeux -# channel latrois -# channel laune -# channel lci -# channel lcp24 -# channel lequipe21 -# channel luckyjacktv -# channel m6 -# channel m6music -# channel mangas -# channel mcetv -# channel mcm -# channel mcmpop -# channel mcmtop -# channel mcs -# channel mcsbienetre -# channel mcsextreme -# channel mensup -# channel mezzo -# channel mezzolivehd -# channel montagnetv -# channel motorstv -# channel mtv -# channel mtvbase -# channel mtvhits -# channel mtvidol -# channel mtvpulse -# channel myzentv -# channel nationalgeographic -# channel nationalgeographicwild -# channel nickelodeon -# channel nickelodeonjunior -# channel nolife -# channel nrj12 -# channel nrjhits -# channel nt1 -# channel numero23 -# channel ocschoc -# channel ocscity -# channel ocsgeants -# channel ocsmax -# channel ofivetv -# channel oltv -# channel paramountchannel -# channel parispremiere -# channel piwiplus -# channel planeteplus -# channel planeteplusae -# channel planeteplusci -# channel planeteplusthalassa -# channel plugrtl -# channel publicsenat24 -# channel publicsenatlcpan -# channel rmcdecouverte -# channel rtl9 -# channel rtltvi -# channel rtsdeux -# channel rtsun -# channel seasonsplus -# channel serieclub -# channel syfy -# channel telegrenoble -# channel teletoonplus -# channel teva -# channel tf1 -# channel tiji -# channel tmc -# channel toutelhistoire -# channel tracesports -# channel tracetropical -# channel traceurban -# channel tv5mondefbs -# channel tv8montblanc -# channel tvbreizh -# channel ushuaiatv -# channel vh1 -# channel vh1classic -# channel vivolta -# channel voyage -# channel w9 +cachedir=/tmp/.xmltv/cache +bouquet=grandes-chaines-et-tnt +channel!6ter.telestar.fr +channel!arte.telestar.fr +channel!bfmtv.telestar.fr +channel!c8.telestar.fr +channel=canalplus.telestar.fr +channel!cherie25.telestar.fr +channel!cnews.telestar.fr +channel!cstar.telestar.fr +channel=france2.telestar.fr +channel!france3.telestar.fr +channel!france4.telestar.fr +channel!france5.telestar.fr +channel!franceo.telestar.fr +channel!gulli.telestar.fr +channel!lequipe.telestar.fr +channel!m6.telestar.fr +channel!nrj12.telestar.fr +channel!publicsenatlcpan.telestar.fr +channel!rmcdecouverte.telestar.fr +channel!rmcstory.telestar.fr +channel=tf1.telestar.fr +channel!tf1seriesfilms.telestar.fr +channel!tfx.telestar.fr +channel!tmc.telestar.fr +channel!w9.telestar.fr diff -Nru xmltv-0.6.1/grab/fr/tv_grab_fr xmltv-0.6.3/grab/fr/tv_grab_fr --- xmltv-0.6.1/grab/fr/tv_grab_fr 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fr/tv_grab_fr 2020-09-07 15:02:53.000000000 +0000 @@ -1,612 +1,465 @@ #!/usr/bin/perl -eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}' - if 0; # not running under some shell +use warnings; +use strict; -=head1 NAME +use XMLTV::Ask; +use XMLTV::Capabilities qw/baseline manualconfig cache/; +use XMLTV::Configure::Writer; +use XMLTV::DST; +use XMLTV::Get_nice qw(get_nice_tree); + $XMLTV::Get_nice::ua->parse_head(0); + $XMLTV::Get_nice::FailOnError = 0; +use XMLTV::Memoize; + XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux'); +use XMLTV::Options qw/ParseOptions/; +use XMLTV::ProgressBar; -tv_grab_fr - Grab TV listings for France. +use DateTime; +use DateTime::Duration; +use Encode qw/decode encode/; +use File::Path; +use Getopt::Long; +use HTML::Entities; +use HTML::TreeBuilder; +use HTTP::Cache::Transparent; +use IO::Scalar; -=head1 SYNOPSIS +############################################################################ +# Main declarations # +############################################################################ + +# Grabber details +my $GRABBER_NAME = 'tv_grab_fr'; +my $GRABBER_VERSION = "$XMLTV::VERSION"; - To configure: - tv_grab_fr --configure [--config-file FILE] [--gui OPTION] - To grab listings: - tv_grab_fr [--config-file FILE] [--output FILE] [--days N] - [--offset N] [--quiet] [--debug] - To list available channels: - tv_grab_fr --list-channels - To show capabilities: - tv_grab_fr --capabilities - To show version: - tv_grab_fr --version - Help: - tv_grab_fr --help +my $ROOT_URL = "https://www.telestar.fr"; +my $GRID_FOR_CHANNEL = "$ROOT_URL/programme-tv/"; +my $GRID_FOR_BOUQUET = "$ROOT_URL/programme-tv/bouquets/"; +my $GRID_BY_CHANNEL_PER_DAY = "$ROOT_URL/programme-tv/grille-chaine/"; -=head1 DESCRIPTION +my $ENCODING = "utf-8"; +my $LANG = "fr"; +my $MAX_RETRY = 5; -Output TV listings for many channels available in France (Orange, -Free, cable/ADSL/satellite, Canal+ Sat). The data comes from -telestar.fr. The default is to grab 7 days, but there are usually -14 days of listings available from the current day onwards. +my %tv_attributes = ( + 'source-info-name' => 'Tele Star', + 'source-info-url' => 'telestar.tv', + 'source-data-url' => "$GRID_FOR_CHANNEL", + 'generator-info-name' => "XMLTV/$XMLTV::VERSION, $GRABBER_NAME", +); + +my ( $opt, $conf ) = ParseOptions( { + grabber_name => "$GRABBER_NAME", + version => "$GRABBER_VERSION", + description => "France (Tele Star)", + capabilities => [qw/baseline manualconfig cache apiconfig/], + defaults => { days => 14, offset => 0, quiet => 0, debug => 0, slow => 0 }, + extra_options => [qw/slow/], + stage_sub => \&config_stage, + listchannels_sub => \&list_channels, +} ); + +############################################################################ +# At this point, the script takes over from ParseOptions # +############################################################################ + +validate_options(); +validate_config(); +initialise_cache(); +print_version_info(); + +my $channels = get_configured_channels(1); + +my $writer = setup_xmltv_writer(); +write_xmltv_header($writer); +write_channel_list($writer, $channels); +write_listings_data($writer, $channels); +write_xmltv_footer($writer); + +############################################################################ +# Subroutines # +############################################################################ + +sub config_stage { + my ( $stage, $conf ) = @_; + + my $result; + my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, + encoding => $ENCODING ); + + $writer->start( { grabber => "$GRABBER_NAME" } ); + + if ($stage eq 'start') { + $writer->write_string( { + id => 'cachedir', + title => [ [ "Directory to store $GRABBER_NAME cache", 'en' ] ], + description => [ + [ $GRABBER_NAME . ' uses a cache to store files that have been '. + 'downloaded. Please specify path to cache directory. ', + 'en' ] ], + default => get_default_cachedir(), + } ); + $writer->end('bouquet'); + } + elsif ($stage eq 'bouquet') { + $writer->start_selectone( { + id => 'bouquet', + title => [ [ 'Please select your TV service (bouquet)', 'en' ] ], + description => [ + [ "When choosing which channels to download listings for, $GRABBER_NAME " . + "will only show the channels on your selected TV service.", + 'en' ] ], + } ); + my $bouquets = get_bouquets(); + foreach my $b ( sort keys %$bouquets ) { + my $name = $b; + my $id = $bouquets->{$b}; + + $writer->write_option( { + value => $id, + text => [ [ $name, 'fr' ] ], + } ); + } + $writer->end_selectone(); -Grabbing speed varies, but is typically 1-2 minutes/day/channel. + # The select-channels stage must be the last stage called + $writer->end('select-channels'); + } + else { + die "Unknown stage $stage"; + } -B<--configure> Choose which bouquets/channels to grab listings data for. + return $result; +} -B<--list-channels> List all available channels. +sub list_channels { + my ( $conf, $opt ) = @_; -B<--config-file FILE> Use FILE as config file instead of the default config -file. This allows for different config files for different applications. + # Do not filter channels with --list-channels + my $filtered = $opt->{'list-channels'} ? 0 : 1; -B<--gui OPTION> Use this option to enable a graphical interface to be used. -OPTION may be 'Tk', or left blank for the best available choice. -Additional allowed values of OPTION are 'Term' for normal terminal output -(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar. + my $channels = get_available_channels($opt, $conf, $filtered); -B<--output FILE> Write to FILE rather than standard output. + my $result = ""; + my $fh = new IO::Scalar \$result; + my $oldfh = select( $fh ); -B<--days N> Grab N days (default 7, maximum 14) starting from today. + my %g_args = (OUTPUT => $fh); -B<--offset N> Start grabbing N days from today, rather than starting -today. + my $writer = new XMLTV::Writer(%g_args, encoding => $ENCODING); + $writer->start(\%tv_attributes); -B<--quiet> Suppress the progress messages normally written to standard -error. + foreach my $c_id (sort keys %{$channels}) { + $writer->write_channel($channels->{$c_id}); + } -B<--debug> Provide additional debugging messages during processing. + $writer->end; + select( $oldfh ); + $fh->close(); -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L. + return $result; +} -B<--version> Show the version of the grabber. +sub get_bouquets { + my %bouquets; + debug_print("get_bouquets(): searching for available bouquets"); -B<--help> Print a help message and exit. + my $url = $GRID_FOR_CHANNEL . "bouquets"; + my $t = get_nice_tree($url, undef, undef, undef); + debug_print("get_bouquets(): url = '$url'"); + if (not defined $t) { + print STDERR "Unable to retrieve bouquets page\n"; + return; + } -=head1 SEE ALSO + foreach my $b_tree ( $t->look_down( "_tag", "div", "class", "bouquet" ) ) { + my $b_title = $b_tree->look_down("_tag", "h2")->as_text(); + debug_print(" Found bouquet name: $b_title"); + my $b_url = $b_tree->look_down("_tag", "a", "class", "red-link")->attr('href'); + debug_print(" Found bouquet URL $b_url"); + my ($b_id) = $b_url =~ /^\/programme-tv\/bouquets\/(.+)/; + debug_print(" Found bouquet ID $b_id"); -L + $bouquets{$b_title} = $b_id; + } + $t->delete(); undef $t; -=head1 AUTHOR + return \%bouquets; +} -The current version of the script was rewritten by Nick Morrott, -knowledgejunkie at gmail dot com, to support the new telestar.fr site. +sub get_available_channels { + my ($opt, $conf, $filtered) = @_; -The original author was Sylvain Fabre, centraladmin at lahiette dot com, -with patches from: - - Francois Gouget, fgouget at free dot fr, - - Geoff Westcott, honir999 at gmail dot com, - - Karl Dietz, dekarl at spaetfruehstuecken dot org, - - Niel Markwick, nielm at bigfoot dot com, - - Zubrick, zubrick at number6 dot ch, - - and many more! + my $bouquet_id; -=cut + if ($filtered) { + $bouquet_id = $conf->{'bouquet'}[0]; + if (not defined $bouquet_id) { + debug_print("get_available_channels(): no bouquet specified, please re-configure grabber"); + return; + } + debug_print("get_available_channels(): filtering out unconfigured channels"); + debug_print("get_available_channels(): searching for channels on bouquet ID: $bouquet_id"); + } + else { + debug_print("get_available_channels(): searching all available channels"); + } -# TODO -# ==== -# -# - convert to use ParseOptions -# -# - perhaps we should internationalize messages and docs? -# -# - try to detect the language based on the country so we can make use of the -# set the VO, VF and original title markers -# -# - investigate how to better handle France 5 and Arte as they share a single -# channel for over-the-air broadcasts, but each have their own channel on -# cable, satellite and ADSL. See the thread at: -# http://sourceforge.net/mailarchive/message.php?msg_id=15181920 -# -# - Add caching via HTTP::Cache::Transparent - -use XMLTV::Usage <parse_head(0); -$XMLTV::Get_nice::FailOnError = 0; + my %available_channels; -use XMLTV::Memoize; -XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux'); + BOUQUET: + foreach my $b_tree ( $t->look_down( "_tag", "div", "class", "bouquet" ) ) { + my $b_title = $b_tree->look_down("_tag", "h2")->as_text(); + my $b_url = $b_tree->look_down("_tag", "a", "class", "red-link")->attr('href'); + my ($b_id) = $b_url =~ /^\/programme-tv\/bouquets\/(.+)/; + next BOUQUET unless (!$filtered || ($b_id eq $bouquet_id)); + debug_print("get_available_channels(): found requested bouquet ID: $b_id"); + + CHANNEL: + my @b_chans = $b_tree->look_down( "_tag", "a" ); + debug_print(" Found " . scalar @b_chans . " channels"); + foreach my $b_chan (@b_chans) { + my $c_url = $b_chan->attr('href'); + if ( $c_url =~ /^\/programme-tv\/grille-chaine\/(.+)/ ) { + my $c_name = $b_chan->as_text(); + my $c_id = $1; + debug_print(" available channel: $c_name ($c_id)"); + + my %ch = ( + 'id' => $c_id . ".telestar.fr", + 'display-name' => [[ $c_name, 'fr' ]], + ); -use DateTime; -use DateTime::Duration; -use Encode qw(decode_utf8 encode_utf8); -use Getopt::Long; -use HTML::Entities; -use HTML::TreeBuilder; -use IO::File; + $available_channels{$c_id} = \%ch; + } + } + } + $t->delete(); undef $t; -#*************************************************************************** -# Main declarations -#*************************************************************************** -my $ROOT_URL = 'http://www.telestar.fr'; -my $GRID_FOR_CHANNEL = "$ROOT_URL/programme-tv/"; -my $GRID_BY_CHANNEL_PER_DAY = "$ROOT_URL/programme-tv/grille-chaine/"; + return \%available_channels; +} -my $ENCODING = 'utf-8'; -my $LANG = "fr"; -my $MAX_RETRY = 5; +sub setup_xmltv_writer { + my %g_args = (); + if (defined $opt->{output}) { + debug_print("\nOpening XMLTV output file '$opt->{output}'\n"); + my $fh = new IO::File ">$opt->{output}"; + die "Error: Cannot write to '$opt->{output}', exiting" if (! $fh); + %g_args = (OUTPUT => $fh); + } -my %GridType = ( "Grandes Chaines" => "grandes-chaines-et-tnt", - "Orange" => "orange", - "Free" => "free", - "Bouygues" => "bouygues", - "SFR" => "sfr", - "Numéricable" => "numericable", - "Canal+" => "canal", - "Câble, ADSL, Satellite" => "cable-adsl-satellite", - "Canal+ et Canalsat" => "canal-et-canalsat", - "Belgique" => "belgique", - "Hors bouquet" => "hors-bouquet", - ); - -#*************************************************************************** -# Global variables allocation according to options -#*************************************************************************** - -my ( $opt_config_file, $opt_days, $opt_offset, $opt_output, - $opt_gui, $opt_quiet, $opt_list_channels, $opt_configure, - $opt_help, $opt_debug, ); - -# Default to non-quiet, non-debug mode -$opt_quiet = 0; -$opt_debug = 0; - -# Although the website is able to provide up to fourteen days of listings, -# parsing pages is slow so we set the default grab to 7 days. You can -# specify up to 14 days via the --days option. -my $default_opt_days = 7; - -# Default to STDOUT -$opt_output = '-'; - -GetOptions( 'days=i' => \$opt_days, - 'help' => \$opt_help, - 'output=s' => \$opt_output, - 'offset=i' => \$opt_offset, - 'quiet' => \$opt_quiet, - 'configure' => \$opt_configure, - 'gui:s' => \$opt_gui, - 'debug' => \$opt_debug, - 'config-file=s' => \$opt_config_file, - 'list-channels' => \$opt_list_channels, - ) - or usage(0); - -#*************************************************************************** -# Options processing, warnings, checks and default parameters -#*************************************************************************** -die 'Number of days must not be negative' if (defined $opt_days && $opt_days < 0); -die 'Number of days must be less than 15' if (defined $opt_days && $opt_days > 14); -die 'Offset must not be negative' if (defined $opt_offset && $opt_offset < 0); -die 'Offset must be less than 14' if (defined $opt_offset && $opt_offset > 13); -die 'Offset+days must be less than 15' if (defined $opt_offset && defined $opt_days && ($opt_offset + $opt_days > 14)); - -die 'Error: You cannot specify --quiet with --debug, exiting' if ($opt_quiet && $opt_debug); - -usage(1) if $opt_help; - -XMLTV::Ask::init($opt_gui); - -# The days/offset options can be used, but we default them if not set. -$opt_offset = 0 if not defined $opt_offset; -$opt_days = $default_opt_days if not defined $opt_days; - -if ( (($opt_offset + $opt_days) > $default_opt_days) or ($opt_offset > $default_opt_days) ) { - $opt_days = $default_opt_days - $opt_offset; - if ($opt_days < 0) { - $opt_offset = 0; - $opt_days = $default_opt_days; - } - say < 'configure', - $opt_list_channels => 'list-channels' - ); - -# File that stores which channels to download -my $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_fr', $opt_quiet); - -#*************************************************************************** -# Subroutines -#*************************************************************************** -sub get_dates_to_grab; -sub get_listings_page_start_date; -sub get_data_for_available_days; -sub get_data_for_filtered_days; -sub process_channel_row; -sub process_program; -sub get_channels; -sub debug_print; -sub trim; -sub tidy_html; -sub parse_name_list; -sub get_tree; -sub get_country_code; -sub get_date_today; -sub get_date_today_with_offset; - -#*************************************************************************** -# Configure mode -#*************************************************************************** -if ($mode eq 'configure') { - XMLTV::Config_file::check_no_overwrite($config_file); - open(CONF, ">$config_file") or die "Cannot write to $config_file: $!"; - - # Choose which bouquets to configure channels for - my @gts = sort keys %GridType; - my @gtnames = map { $GridType{$_} } @gts; - my @gtqs = map { "Get channels type : $_?" } @gts; - my @gtwant = ask_many_boolean(1, @gtqs); - - # Get channels for each requested bouquet - my $bar = new XMLTV::ProgressBar('getting channel lists', - scalar grep { $_ } @gtwant) - if not $opt_quiet; - my %channels_for; - foreach my $i (0 .. $#gts) { - my ($gt, $gtw, $gtname) = ($gts[$i], $gtwant[$i], $gtnames[$i]); - next if not $gtw; - my %channels = get_channels( $gtname ); - print STDERR 'WARNING: No channels could be found'."\n" if not %channels; - $channels_for{$gt} = \%channels; - update $bar if not $opt_quiet; - } - $bar->finish() if not $opt_quiet; - - my %asked; - foreach (@gts) { - my $gtw = shift @gtwant; - my $gtname = shift @gtnames; - if ($gtw) { - my %channels = %{$channels_for{$_}}; - say "Channels for $_"; - - # Ask about each channel (unless already asked). - my @chs = grep { not $asked{$_}++ } sort keys %channels; - my @names = map { $channels{$_}{name} } @chs; - my @qs = map { "add channel $_?" } @names; - my @want = ask_many_boolean(1, @qs); - foreach (@chs) { - my $w = shift @want; - warn("cannot read input, stopping channel questions"), last if not defined $w; - # Print a config line, but comment it out if channel not wanted. - print CONF '#' if not $w; - print CONF "channel $channels{$_}{name}\n"; - } + return new XMLTV::Writer(%g_args, encoding => $ENCODING); +} + +sub write_xmltv_header { + my $writer = shift; + debug_print("Writing XMLTV header"); + $writer->start(\%tv_attributes); +} + +sub write_channel_list { + my ($writer, $channels) = @_; + + debug_print("write_channel_list: writing elements"); + foreach my $c_id (sort keys %{$channels}) { + my $c_name = encode_and_trim( $channels->{$c_id}{'display-name'}[0][0]); + my %ch = ( + 'id' => $c_id . ".telestar.fr", + 'display-name' => [[ $c_name, 'fr' ]], + ); + $writer->write_channel( \%ch ); + } +} + +sub get_configured_channels { + my $filtered = shift; + my $available_channels = get_available_channels($opt, $conf, $filtered); + + my %seen_ids; + foreach (keys %{$available_channels}) { + $seen_ids{$_} = 0; + } + + debug_print("get_configured_channels(): checking configured channels"); + foreach my $c_id (@{$conf->{'channel'}}) { + ($c_id) = $c_id =~ /^(\w+)\.telestar\.fr$/; + if (! exists $seen_ids{$c_id}) { + debug_print("** UNAVAILABLE channel: '$c_id'"); + } + else { + my $c_name = $available_channels->{$c_id}{'display-name'}[0][0]; + debug_print(" configured channel: $c_name ($c_id)"); + $seen_ids{$c_id} = 1; } } - close CONF or warn "cannot close $config_file: $!"; - say("Finished configuration."); - exit(); -} - -#*************************************************************************** -# Check mode -#*************************************************************************** -die if $mode ne 'grab' and $mode ne 'list-channels'; - -#*************************************************************************** -# Prepare the XMLTV writer object -#*************************************************************************** -my %w_args; -if (defined $opt_output) { - my $fh = new IO::File(">$opt_output"); - die "cannot write to $opt_output: $!" if not defined $fh; - $w_args{OUTPUT} = $fh; -} - -$w_args{encoding} = "$ENCODING"; -$w_args{days} = "$opt_days"; -$w_args{offset} = "$opt_offset"; -$w_args{cutoff} = "000000"; - -my $writer = new XMLTV::Writer(%w_args); -$writer->start - ({ 'source-info-url' => "$ROOT_URL/", - 'source-data-url' => "$ROOT_URL/", - 'generator-info-name' => 'XMLTV', - 'generator-info-url' => 'http://xmltv.org/', - }); - -#*************************************************************************** -# List channels only case -#*************************************************************************** -if ($mode eq 'list-channels') { - # Get a list of all available channels, for all bouquets - # - # Unlike channel configuration in --configure, we do not ask the user - # for any input to select bouquets, we list everything - my @gts = sort keys %GridType; - my @gtnames = map { $GridType{$_} } @gts; - my %seen; - foreach (@gts) { - my $gtname = shift @gtnames; - my %channels = get_channels( $gtname ); - print STDERR 'WARNING: No channels could be found'."\n" if (scalar(keys(%channels)) == 0); - foreach my $ch_did (sort(keys %channels)) { - my $ch_xid = $ch_did.".telestar.fr"; - $writer->write_channel({ id => $ch_xid, - 'display-name' => [[encode_utf8( $channels{$ch_did}{name} )]], - }) - unless $seen{$ch_xid}++; - } - } - $writer->end(); - exit(); -} - -#*************************************************************************** -# Only grabbing beyond here... -#*************************************************************************** -die if $mode ne 'grab'; - -#*************************************************************************** -# Read configuration file -#*************************************************************************** -my @config_lines; -@config_lines = XMLTV::Config_file::read_lines($config_file); - -my (%channels, $chname); -my $line_num = 1; -foreach (@config_lines) { - ++ $line_num; - next if not defined; - - # We store the channel name as the identifier in the config file - if (/^channel (\w+)$/) { - $chname = $1; - $channels{$line_num} = { 'name'=>$chname }; - } else { - warn "$config_file:$line_num: bad line $_\n"; - } -} - -#*************************************************************************** -# Process requested days -#*************************************************************************** -warn "No working channels configured, so no listings\n" if not %channels; -my $script_duration = time(); - -my @to_get; -my $listings_page_start_date = get_listings_page_start_date(); - -# The telestar.fr website provides up to 14 days of listings for each -# channel. The first 7 days from today are on one page, the second -# 7 days on another. -debug_print("\nCreating list of pages to grab based on configured channels..."); -foreach my $idx (sort { $a <=> $b } keys %channels) { - my $chname = $channels{$idx}{name}; - my $xmltvid = $chname.".telestar.fr"; - my $url; - my $dayoff; - - $writer->write_channel({ id => $xmltvid, - 'display-name' => [[encode_utf8( $chname )]], - }); - - debug_print("\n : xmltvid=$xmltvid, name=$chname"); - debug_print(" Creating list of pages for " . $chname); - - my @urls; - if ($opt_offset < 7) { - $url = $GRID_BY_CHANNEL_PER_DAY . $chname; - push @urls, $url; - debug_print( " adding: url=$url" ); - } - if ($opt_offset + $opt_days >= 7) { - $dayoff = get_date_today_with_offset(7)->strftime("%d-%m-%Y"); - $url = $GRID_BY_CHANNEL_PER_DAY . "$chname/$dayoff"; - push @urls, $url; - debug_print( " adding: url=$url" ); - } - push @to_get, [\@urls, $chname]; -} - -debug_print("\nGenerating list of dates...\n"); -my $dates_to_grab = get_dates_to_grab(); - -my $bar; -if (not $opt_quiet and not $opt_debug) { - $bar = new XMLTV::ProgressBar('getting listings', scalar @to_get); -} - -debug_print("\nProcessing list of pages to grab...\n"); -CHANNEL: -foreach (@to_get) { - # Each item in the @to_get listref is a listref for each channel - # (a list of URLS to get, and the channel name). - my ($urls, $chname) = @$_; - my @data_for_available_days; - - foreach my $url (@$urls) { - push @data_for_available_days, get_data_for_available_days($url, $chname); - } - - my $data_for_filtered_days = get_data_for_filtered_days(\@data_for_available_days); - - debug_print(" Have " . scalar @{$data_for_filtered_days} . " days of data\n"); - debug_print(" Have " . scalar @{$dates_to_grab} . " dates to grab for\n"); - - if (scalar @{$data_for_filtered_days} != scalar @{$dates_to_grab}) { - if (not $opt_quiet) { - print STDERR "ERROR: Not enough data available for requested number of days\n"; - print STDERR " Have " . scalar @{$data_for_filtered_days} . " days of data\n"; - print STDERR " Need " . scalar @{$dates_to_grab} . " days of data\n"; - print STDERR " Skipping channel\n"; + + # remove any channels not flagged + my %available_configured; + foreach my $c_id (keys %{$available_channels}) { + if ($seen_ids{$c_id}) { + $available_configured{$c_id} = $available_channels->{$c_id}; } - next CHANNEL; } - # Store individual programmes in a list and write each channel in full. - # No stop times are given in the listings (only inaccurate durations), so - # we can use the start time of a following programme as the stop time of - # the previous programme. (May fail if channel does not have listings - # for full 24hrs). - my @programmes = (); - foreach my $i (0 .. (scalar @{$data_for_filtered_days} - 1)) { - debug_print(" Getting listings for day " . ($i+1)); - push @programmes, @{ process_channel_row( $chname, $data_for_filtered_days->[$i], $dates_to_grab->[$i] ) }; + if ($opt->{'debug'}) { + my $wanted = scalar @{$conf->{'channel'}}; + my $actual = scalar keys %available_configured; + debug_print("get_configured_channels(): $actual/$wanted configured channels supported by grabber"); } - # Update programme stop times - @programmes = @{ update_programme_stop_times(\@programmes) }; + return \%available_configured; +} + +sub write_listings_data { + my ($writer, $channels) = @_; - # Write the channel's programme elements to output - foreach my $prog (@programmes) { - $writer->write_programme($prog); - } + my $dates = get_dates_to_grab(); + my $urls = generate_urls_to_grab($dates); - if (not $opt_quiet and not $opt_debug) { - update $bar; + my $bar; + if (not $opt->{quiet} and not $opt->{debug}) { + $bar = new XMLTV::ProgressBar('Getting listings...', scalar keys %$urls); } -} -$writer->end(); + # Store individual programmes in a list and write each channel in full later. + # key = upstream channel ID, value = listref of programme elements + my %programmes; -if (not $opt_quiet and not $opt_debug) { - $bar->finish(); -} + debug_print("\nProcessing list of URLs to grab...\n"); + foreach my $ymd (sort keys %$urls) { + + my $url = $urls->{$ymd}; + my $progs_on_channel = get_daily_data_for_requested_channels($url, $ymd, $channels); + + foreach my $c (keys %$progs_on_channel) { + push @{ $programmes{$c} }, @{ $progs_on_channel->{$c} }; + } + + if (not $opt->{quiet} and not $opt->{debug}) { + update $bar; + } + } -# Print the script's execution time -$script_duration = time() - $script_duration; -print STDERR "Grabber process finished in " . $script_duration . " seconds.\n" if not $opt_quiet; + # use Data::Dumper; print STDERR Dumper(\%programmes); + # exit; -#*************************************************************************** -# Subroutines -#*************************************************************************** + # No stop times are given in the listings (only inaccurate durations), so + # we can use the start time of a following programme as the stop time of + # the previous programme. (May fail if channel does not have listings + # for full 24hrs). + foreach my $c (keys %programmes) { + debug_print(" Analysing/updating schedule gaps between programmes on channel ID '$c'"); + $programmes{$c} = update_programme_stop_times($programmes{$c}); + } + + # Write out all available programme elements for each channel + foreach my $c_id (sort keys %programmes) { + debug_print(" Writing listings for channel '$c_id'"); + foreach my $p (@{$programmes{$c_id}}) { + $writer->write_programme($p); + } + } +} sub get_dates_to_grab { my @dates_to_grab = (); # First date to grab listings for - my $grab_start_date = get_date_today_with_offset($opt_offset); + my $grab_start_date = get_date_today_with_offset($opt->{offset}); push @dates_to_grab, $grab_start_date; # Remaining dates to grab listings for - for (my $offset = 1; $offset < $opt_days; $offset++) { + for (my $offset = 1; $offset < $opt->{days}; $offset++) { push @dates_to_grab, $grab_start_date + DateTime::Duration->new( days => $offset ); } debug_print("Will grab listings for following dates:"); - if ($opt_debug) { + if ($opt->{debug}) { foreach (@dates_to_grab) { - # print STDERR " " . $_->strftime("%a, %d %b %Y %H:%M:%S %z") . "\n"; print STDERR " " . $_->strftime("%a, %d %b %Y") . "\n"; } } - debug_print("\n"); return \@dates_to_grab; } -# Each week of listings for a channel is provided on a single page. -# -# * if we are grabbing with no starting offset (today) or an offset less than -# 7 days into the future, we'll start with the default page (starts today). -# -# * if we are grabbing with a starting offset of at least 7 days, we'll -# start with next week's page. -# -sub get_listings_page_start_date { - my $start_date; - - if ($opt_offset < 7) { - $start_date = get_date_today(); - debug_print("get_listings_page_start_date(): " . $start_date->strftime("%d-%m-%Y")); - } - else { - $start_date = get_date_today_with_offset(7); - debug_print("get_listings_page_start_date(): " . $start_date->strftime("%d-%m-%Y")); +sub generate_urls_to_grab { + my ($dates_to_grab) = @_; + my $bouquet = $conf->{'bouquet'}[0]; + my %urls; + + debug_print("Creating list of URLs to grab based on configured bouquet..."); + foreach my $d (@$dates_to_grab) { + my $ymd = $d->strftime("%Y%m%d"); + my $dmy = $d->strftime("%d-%m-%Y"); + my $url = $GRID_FOR_BOUQUET . $bouquet . "/journee/(date)/" . $dmy . "/(ajax)/1"; + $urls{$ymd} = $url; + debug_print( " Adding URL: $url" ); } - return $start_date; + return \%urls; } -sub get_data_for_available_days { - my ($url, $chname) = @_; +sub get_daily_data_for_requested_channels { + my ($url, $ymd, $channels) = @_; - debug_print("get_available_days(): url=$url"); + debug_print("get_daily_data_for_requested_channels(): url=$url"); # Get the page's tree - my $t = get_tree($url); + my $t = get_nice_tree($url, undef, undef, undef); if (not defined $t) { - debug_print("*** Error: Could not get tree for '" . $url . "' ***"); + debug_print("Error: Could not get data for URL: $url"); return; } - # Return all available day elements (
    ) - my @nodes = $t->look_down('_tag', 'div', 'class', 'channel'); - debug_print("get_available_days(): Found " . scalar @nodes . " days of data\n"); - return @nodes; -} - -sub get_data_for_filtered_days { - my $available_days = shift; - my @filtered_days; - - AVAILABLE: - foreach my $i (0 .. scalar @$available_days) { - next AVAILABLE if ($i < $opt_offset); - last AVAILABLE if ($i == $opt_offset + $opt_days); - push @filtered_days, $available_days->[$i]; - } - - debug_print("filter_days(): Will get listings for " . scalar @filtered_days . " days\n"); - undef $available_days; - return \@filtered_days; + # Locate the listings grid + my $grid = $t->look_down('_tag', 'div', 'class', 'grid-content'); + + # Locate the channel container in the grid and list of available channels + my $c_cont = $grid->look_down('_tag', 'div', 'id', 'channels'); + my @available_channels = $c_cont->look_down('_tag', 'div', 'class', 'channel'); + + # Locate the programme container in the grid + my $p_cont = $grid->look_down('_tag', 'div', 'id', 'programs'); + my @programmes = $p_cont->look_down('_tag', 'div', 'class', 'channel'); + + my %progs_on_channel; + + foreach my $i (0 .. (scalar @available_channels -1)) { + my $c = $available_channels[$i]; + my $c_url = $c->look_down('_tag', 'a')->attr('href'); + my ($c_id) = $c_url =~ /^\/programme-tv\/grille-chaine\/(.+)/; + if (exists($channels->{$c_id})) { + $progs_on_channel{$c_id} = process_channel_row($c_id, $programmes[$i], $ymd); + } + } + + $t->delete(); undef $t; + + return \%progs_on_channel; } sub process_channel_row { - my ($chname, $row, $dt) = @_; - - my $dateindex = $dt->strftime("%Y%m%d"); + my ($c_id, $row, $ymd) = @_; - debug_print(" ###############################################################"); - debug_print(" process_row: processing listings for: chname=$chname, dateindex=$dateindex"); - debug_print(" ###############################################################\n"); + debug_print("process_channel_row: processing listings for: $c_id ($ymd)"); my @programmes = (); PROGRAMME: foreach my $programme ($row->look_down('_tag', 'div', 'class', qr/program /) ) { # skip empty program cells if ($programme->attr('class') =~ /no-program/) { - debug_print(" Skipping 'no-program' entry\n"); + debug_print(" skipping 'no-program' entry\n"); next PROGRAMME; } # extract the programme data - push @programmes, process_program($chname, $programme, $dateindex); + push @programmes, process_program($c_id, $programme, $ymd); debug_print("\n"); } @@ -614,7 +467,7 @@ } sub process_program { - my ($chname, $programme, $dateindex) = @_; + my ($c_id, $programme, $ymd) = @_; my $title_text; my $prog_page; @@ -622,9 +475,8 @@ if ($title) { if ($title->as_text() =~ /\w+/) { $title_text = trim($title->as_text()); - debug_print(" Found programme title '" . $title_text . "'"); + debug_print("process_program: found programme title '" . $title_text . "'"); - #FIXME for all prog types my $link = $title->look_down('_tag', 'a', 'class', 'lien-fiche'); if ($link and $link->attr('href') =~ /programme-tv/) { $prog_page = $ROOT_URL . $link->attr('href'); @@ -637,7 +489,7 @@ } } else { - debug_print(" No programme title found, skipping programme"); + debug_print(" No programme title tag found, skipping programme"); return undef; # REQUIRED } @@ -647,7 +499,7 @@ if ($start) { if ($start->as_text() =~ /(\d\d)h(\d\d)/) { my ($hh, $mm) = ($1, $2); - $start_time = $dateindex.$hh.$mm."00"; + $start_time = $ymd.$hh.$mm."00"; debug_print(" Found programme start '" . $hh."h".$mm . "'"); } else { @@ -674,8 +526,8 @@ } debug_print(" Creating programme hash for '" . $title_text . " / " . $start_time); - my %prog = (channel => $chname.".telestar.fr", - title => [ [ encode_utf8( trim($title_text) ) ] ], # lang unknown + my %prog = (channel => $c_id.".telestar.fr", + title => [ [ encode_and_trim($title_text), $LANG ] ], start => utc_offset($start_time, "+0100"), ); @@ -708,23 +560,23 @@ } $episodenumber = $episodenumber->as_text(); $prog{'episode-num'} = [ [ $season_num . "." . $episode_num . ".", "xmltv_ns" ] ]; - debug_print(" Found programme episodenumber '" . $episodenumber . "'"); + debug_print(" Found programme episode numbering '" . $episodenumber . "'"); } # Likely the programme's sub-title if not an episode number elsif ($episodenumber->as_text() =~ /\w+/) { $episodenumber = $episodenumber->as_text(); - $prog{'sub-title'} = [ [ encode_utf8( $episodenumber ) ] ]; + $prog{'sub-title'} = [ [ encode_and_trim( $episodenumber ), $LANG ] ]; debug_print(" Found programme sub-title '" . $episodenumber . "'"); } } else { - debug_print(" No episodenumber found"); + debug_print(" No episode numbering found"); } my $category = $programme->look_down('_tag', 'p', 'class', 'category'); if ($category and $category->as_text() =~ /\w+/) { $category = trim($category->as_text()); - $prog{category} = [ [ encode_utf8( $category ), $LANG ] ]; + $prog{category} = [ [ encode_and_trim($category), $LANG ] ]; debug_print(" Found programme genre '" . $category . "'"); } else { @@ -734,8 +586,8 @@ my $synopsis = $programme->look_down('_tag', 'p', 'class', 'synopsis'); if ($synopsis and $synopsis->as_text() =~ /\w+/) { $synopsis = trim($synopsis->as_text()); - $prog{desc} = [ [ encode_utf8( $synopsis ), $LANG ] ]; - debug_print(" Found programme synopsis '" . $synopsis . "'"); + $prog{desc} = [ [ encode_and_trim($synopsis), $LANG ] ]; + debug_print(" Found programme short synopsis '" . $synopsis . "'"); } else { debug_print(" No synopsis found"); @@ -758,193 +610,309 @@ debug_print(" Found programme icon: '" . $url . "'"); } - if ($prog_page) { + if ($opt->{'slow'} && $prog_page) { process_programme_page(\%prog); } return \%prog; } -# Process a page of detailed programme information for stored url and -# update the given programme hash with any extra information extracted -# from the page -# sub process_programme_page { my $prog = shift; my $prog_page = $prog->{'_prog_page'}; - debug_print(" process_programme_page(): url=$prog_page"); + debug_print("process_programme_page(): $prog_page"); # Get the page's tree - my $t = get_tree($prog_page); + my $t = get_nice_tree($prog_page, undef, undef, undef); if (not defined $t) { debug_print(" *** Error: Could not get tree for '" . $prog_page . "' ***"); return $prog; } - my $prog_info = $t->look_down('_tag', 'div', 'class', qr/program-informations/); - if ($prog_info) { - my $episode_name = $prog_info->look_down('_tag', 'h2', 'class', 'underlined red'); - if ($episode_name) { - if ($episode_name->as_text() =~ /Saison \d+ Episode \d+ : (\w.*)$/) { - $episode_name = trim($1); - $prog->{'sub-title'} = [ [ encode_utf8( $episode_name ), $LANG ] ]; - debug_print(" Found programme sub-title '" . $episode_name . "'"); - } - elsif ($episode_name->as_text() =~ /\w+/) { - $episode_name = trim($episode_name->as_text()); - $prog->{'sub-title'} = [ [ encode_utf8( $episode_name ), $LANG ] ]; - debug_print(" Found programme sub-title '" . $episode_name . "'"); - } - } - my $date_created = $prog_info->look_down('_tag', 'span', 'itemprop', 'dateCreated'); - if ($date_created) { - if ($date_created->as_text() =~ /^\d{4}$/) { - $date_created = trim($date_created->as_text()); - $prog->{'date'} = $date_created; - debug_print(" Found programme year '" . $date_created . "'"); - } - } - my $genre = $prog_info->look_down('_tag', 'span', 'itemprop', 'genre'); - if ($genre) { - my $subgenre; - ($genre, $subgenre) = split(/,|\s-\s/, $genre->as_text() ); - if (defined $genre && $genre =~ /\w+/) { - $genre = trim($genre); - debug_print(" Found programme genre '" . $genre . "'"); - - if (defined $subgenre && $subgenre =~ /\w+/) { - $subgenre = trim($subgenre); - debug_print(" Found programme sub-genre '" . $subgenre . "'"); + # constrain searching to main content pane + my $c = $t->look_down('_tag', 'div', 'class', qr/content left/); - $prog->{category} = [ [ encode_utf8( $genre ), $LANG ], - [ encode_utf8( $subgenre ), $LANG ] ]; + my $prog_info = $c->look_down('_tag', 'ul', 'class', 'list-fiche'); + if ($prog_info) { + my @info_fields = $prog_info->look_down('_tag', 'li'); + if (@info_fields) { + # each info field comprises 2 tags giving a key and a value + foreach my $info_field (@info_fields) { + if ($info_field->as_text() =~ /^Titre : (.+)$/) { + my $episode_name = trim($1); + $prog->{'sub-title'} = [ [ encode_and_trim( $episode_name ), $LANG ] ]; + debug_print(" Found programme sub-title: " . $episode_name); + } + elsif ($info_field->as_text() =~ /de production : (\d{4})$/) { + my $date_created = trim($1); + $prog->{'date'} = $date_created; + debug_print(" Found production year: " . $date_created); } - else { - $prog->{category} = [ [ encode_utf8( $genre ), $LANG ] ]; + elsif ($info_field->as_text() =~ /^Genre : (.+)$/) { + my $genre = trim($1); + my $subgenre; + ($genre, $subgenre) = split(/,|\s-\s/, $genre); + if (defined $genre && $genre =~ /\w+/) { + $genre = trim($genre); + debug_print(" Found programme genre: " . $genre); + + if (defined $subgenre && $subgenre =~ /\w+/) { + $subgenre = trim($subgenre); + debug_print(" Found programme sub-genre: " . $subgenre); + + $prog->{category} = [ [ encode_and_trim( $subgenre ), $LANG ], + [ encode_and_trim( $genre ), $LANG ] ]; + } + else { + $prog->{category} = [ [ encode_and_trim( $genre ), $LANG ] ]; + } + } } } } } - my $synopsis = $t->look_down('_tag', 'p', 'class', qr/synopsis/); - if ($synopsis) { - $synopsis = trim($synopsis->as_text()); - $prog->{desc} = [ [ encode_utf8( $synopsis ), $LANG ] ]; - debug_print(" Found programme synopsis '" . $synopsis . "'"); + + my $title_block = $c->look_down('_tag', 'div', 'class', qr/title-block/); + if ($title_block) { + # Remove
    containing 'Synopsis' text + my $parent = $title_block->parent(); + $title_block->delete(); + # Process remaining text + my $synopsis = trim($parent->as_text()); + $prog->{desc} = [ [ encode_and_trim( $synopsis ), $LANG ] ]; + debug_print(" Found programme long synopsis: " . $synopsis); } # Casting information on the default programme information page is # typically limited to series. - # - # A separate "Casting" page is available for many programmes which may - # include casting information for the given programme, but typically - # includes all casting information for the whole series of programmes so is - # not currently used. - my $casting = $t->look_down('_tag', 'div', 'class', qr/block-casting/); + my $casting = $c->look_down('_tag', 'div', 'id', 'block-casting', 'class', 'block-casting'); if ($casting) { - my @directors = $casting->look_down('_tag', 'a', 'itemprop', 'producer'); - foreach my $director (@directors) { - $director = trim($director->as_text()); - push @{$prog->{credits}{director}}, encode_utf8( $director ); - debug_print(" Found programme director '" . $director . "'"); - } - - my @actors = $casting->look_down('_tag', 'a', 'itemprop', 'actor'); - foreach my $actor (@actors) { - $actor = trim($actor->as_text()); - push @{$prog->{credits}{actor}}, encode_utf8( $actor ); - debug_print(" Found programme actor '" . $actor . "'"); - } - - my @writers = $casting->look_down('_tag', 'a', 'itemprop', 'author'); - foreach my $writer (@writers) { - $writer = trim($writer->as_text()); - push @{$prog->{credits}{writer}}, encode_utf8( $writer ); - debug_print(" Found programme writer '" . $writer . "'"); + my @casting_titles; + @casting_titles = $casting->look_down('_tag', 'h3', 'class', 'title'); + # some page styles (fiche-emission) may use h4 instead + unless (@casting_titles) { + @casting_titles = $casting->look_down('_tag', 'h4', 'class', 'title'); + } + foreach my $ct (@casting_titles) { + if ($ct->as_text() =~ /R.alisateur/) { + my $parent = $ct->parent(); + my @directors = $parent->look_down('_tag', 'span', 'class', 'name'); + foreach my $director (@directors) { + $director = trim($director->as_text()); + push @{$prog->{credits}{director}}, encode_and_trim( $director ); + debug_print(" Found programme director: " . $director); + } + } + elsif ($ct->as_text() =~ /Sc.nario/) { + my $parent = $ct->parent(); + my @writers = $parent->look_down('_tag', 'span', 'class', 'name'); + foreach my $writer (@writers) { + $writer = trim($writer->as_text()); + push @{$prog->{credits}{writer}}, encode_and_trim( $writer ); + debug_print(" Found programme writer: " . $writer); + } + } + elsif ($ct->as_text() =~ /Acteurs et actrices/) { + my $parent = $ct->parent(); + my @actors = $parent->look_down('_tag', 'span', 'class', 'name'); + foreach my $actor (@actors) { + $actor = trim($actor->as_text()); + push @{$prog->{credits}{actor}}, encode_and_trim( $actor ); + debug_print(" Found programme actor: " . $actor); + } + } } } + $c->delete(); undef $c; + $t->delete(); undef $t; + return $prog; } -# For each programme in the given programme listref, set the stop time -# of a given programme to the start time of the following programme. For -# the last programme in the list, set the stop time to start time plus -# its duration sub update_programme_stop_times { my $programmes = shift; - # Stop at antepenultimate programme + # Stop at penultimate programme foreach my $i (0 .. (scalar @{$programmes} -2)) { - my $prog_current = $programmes->[$i]; - my $prog_next = $programmes->[$i+1]; - - $prog_current->{stop} = $prog_next->{start}; + my $p0 = $programmes->[$i]; + my $p0_stop = get_datetime_from_start_duration($p0); + my $p0_title = decode($ENCODING, $p0->{title}[0][0]); + + my $p1 = $programmes->[$i+1]; + my $p1_start = get_datetime_from_xmltv_time($p1->{start}); + my $p1_title = decode($ENCODING, $p1->{title}[0][0]); + + if ($p1_start == $p0_stop) { + # "This is good..." + debug_print(" No gap detected between '$p0_title' and '$p1_title'"); + $p0->{stop} = $p1->{start}; + } + elsif ($p1_start < $p0_stop) { + # Trust the published start time + if ($opt->{debug}) { + my $dur = $p0_stop - $p1_start; + my $gap = $dur->minutes; + debug_print(" Calculated stop time for '$p0_title' is $gap minutes later than start time of '$p1_title'"); + } + $p0->{stop} = $p1->{start}; + } + elsif ($p1_start > $p0_stop) { + my $dur = $p1_start - $p0_stop; + my $gap = $dur->minutes; + if ($gap <= 10) { + # For small gaps less than 10 minutes, use the next + # programme's start time + debug_print(" There is a small gap of $gap minutes between '$p0_title' and '$p1_title'"); + $p0->{stop} = $p1->{start}; + } + else { + # Otherwise, use the current programme's duration + debug_print(" There is a large gap of $gap minutes between '$p0_title' and '$p1_title'"); + $p0->{stop} = get_xmltv_time_from_datetime($p0_stop); + } + } } - # Handle final programmes separately: add duration to start time - my $prog_last = $programmes->[-1]; - my ($y, $m, $d, $hh, $mm, $ss) = $prog_last->{start} =~ /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/; - my $prog_last_start = DateTime->new( + # Handle final programme separately: add duration to start time + my $p_last = $programmes->[-1]; + my $p_last_stop = get_datetime_from_start_duration($p_last); + $p_last->{stop} = get_xmltv_time_from_datetime($p_last_stop); + + # Return updates listref of programmes + return $programmes; +} + +sub get_datetime_from_start_duration { + my $prog = shift; + + my $dt_start = get_datetime_from_xmltv_time($prog->{'start'}); + my $dt_duration = DateTime::Duration->new( minutes => $prog->{'_duration_mins'}); + + return $dt_start + $dt_duration; +} + +sub get_datetime_from_xmltv_time { + my $date_string = shift; + + my ($y, $m, $d, $hh, $mm, $ss) = $date_string =~ /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/; + my $dt = DateTime->new( year => $y, month => $m, day => $d, hour => $hh, minute => $mm, second => $ss, time_zone => 'Europe/Paris', ); - my $prog_last_duration = DateTime::Duration->new( minutes => $prog_last->{'_duration_mins'}); - my $prog_last_stop = $prog_last_start + $prog_last_duration; - $prog_last->{stop} = utc_offset($prog_last_stop->strftime("%Y%m%d%H%M%S"), "+0100"); - # Return updates listref of programmes - return $programmes; + return $dt; } -# Return a hash of available channels for a given bouquet. -# -sub get_channels { - my $bouquet = shift; - my %bouquet_channels; - - return undef if not defined $bouquet; - - # Get the current page for the given bouquet - # http://www.telestar.fr/programme-tv/grille/09-08-2015/grandes-chaines-et-tnt - my $today = get_date_ddmmyyyy(); - my $url = $GRID_FOR_CHANNEL.'grille/'.$today.'/'.$bouquet; - my $t = get_tree($url); - debug_print("get_channels(): url = '" . $url . "'"); - if (!defined $t) { - print STDERR "Unable to retrieve channels for $bouquet \n"; - return %bouquet_channels; - } - - foreach my $cellTree ( $t->look_down( "_tag", "div", "class", "channel" ) ) { - my $tag = $cellTree->look_down( "_tag", "a" ); - if (defined $tag) { - my $progurl = $tag->attr('href'); - if ( $progurl =~ /^\/programme-tv\/grille-chaine\/(\w+)/ ) { - # There are now no parsable channel names and icons - # are given as a single site-wide imagemap instead of individual URLs: - # http://css1.telestar.fr/extension/telestar/design/telestar/images/chaines/chaines.jpg - my $chname = $1; - debug_print(" Found channel: name = '" . $chname . "'"); - $bouquet_channels{$chname} = {'name' => $chname }; - } +sub get_xmltv_time_from_datetime { + my $dt = shift; + + return utc_offset($dt->strftime("%Y%m%d%H%M%S"), "+0100"); +} + +sub get_date_today { + return DateTime->now( time_zone => 'Europe/Paris' ); +} + +sub get_date_today_with_offset { + my $offset = DateTime::Duration->new( days => shift ); + return get_date_today() + $offset; +} + +sub write_xmltv_footer { + my $writer = shift; + debug_print("\nWriting XMLTV footer\n"); + $writer->end; +} + +sub validate_options { + if ($opt->{quiet} && $opt->{debug}) { + die "Error: You cannot specify --quiet with --debug, exiting"; + } + + if ($opt->{offset} < 0 or $opt->{offset} > 13) { + print STDERR "Invalid value for --offset. Please adjust to a value in range 0-13\n"; + exit 1; + } + + if ($opt->{days} < 1 or $opt->{days} > 14) { + print STDERR "Invalid value for --days. Please adjust to a value in range 1-14\n"; + exit 1; + } + + my $max_days_after_offset = 14 - $opt->{offset}; + if ($opt->{days} > $max_days_after_offset) { + print STDERR "Cannot retrieve more than $max_days_after_offset days of listings\n" . + "Please adjust --days and/or --offset.\n"; + exit 1; + } +} + +sub validate_config { + my @required_keys = ("cachedir", "bouquet", "channel"); + foreach my $key (@required_keys) { + if (! defined $conf->{$key}) { + print STDERR "No configured $key found in config file ($opt->{'config-file'})\n"; + print STDERR "Please reconfigure the grabber ($GRABBER_NAME --configure)\n"; + exit 1; } } - $t->delete(); undef $t; - return %bouquet_channels; } -# Prints a debug string when the --debug option is used -# -sub debug_print { - if ($opt_debug) { - my ($msg) = shift; - print STDERR $msg . "\n"; +sub initialise_cache { + init_cachedir( $conf->{cachedir}->[0] ); + HTTP::Cache::Transparent::init( { + 'BasePath' => $conf->{cachedir}->[0], + 'MaxAge' => 24, + 'NoUpdate' => 60*60*3, + 'Verbose' => $opt->{debug}, + } ); +} + +sub init_cachedir { + my $path = shift; + if (! -d $path) { + mkpath($path) or die "Failed to create cache-directory $path: $@"; } + + debug_print("init_cachedir: cache directory created at $path"); +} + +sub get_default_dir { + my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH} + if (defined $ENV{HOMEDRIVE} and defined $ENV{HOMEPATH}); + + my $home = $ENV{HOME} || $winhome || "."; + + debug_print("get_default_dir: home directory found at $home"); + + return $home; +} + +sub get_default_cachedir { + my $cachedir = get_default_dir() . "/.xmltv/cache"; + + debug_print("get_default_cachedir: default cache directory set to $cachedir"); + + return $cachedir; +} + +sub print_version_info { + debug_print("Program/library version information:\n"); + debug_print("XMLTV library version: $XMLTV::VERSION"); + debug_print(" grabber version: $GRABBER_VERSION"); + debug_print(" libwww-perl version: $LWP::VERSION\n"); +} + +sub encode_and_trim { + my $s = shift; + $s = trim($s); + $s = encode( $ENCODING, $s ); + + return $s; } -# Remove leading/trailing whitespace -# sub trim { for (my $s = shift) { s/^\s*//; @@ -954,71 +922,94 @@ } } -# Replaces specific HTML entities with text replacements, and then -# decodes any remaining entities in the string -# -sub tidy_html { - for (my $s = shift) { - # handle specific entities - s/ / /g; - s/\x8c/Œ/g; - s/Œ/OE/g; - s/\x9c/œ/g; - s/œ/oe/g; - s/“|”|&\#8219;|&\#8220;/\"/g; - s/‘|’|&\#8216;|&\#8217;|&\#8218;/\'/g; - s/&\#8212;/--/g; - s/\x85/.../g; - s/&\#8230;/.../g; - s/&\#8230;/.../g; - s/&\#821[0123];/-/g; - s/\xe2\x80\x99/\'/g; - s/\x92/\'/g; # turn windows-1252 right single quotation mark into apostrophe - s/\x80/€/g; # turn windows-1252 euro sign into real euro sign - s/(&\#[0-9]{4,};)//g; - # decode remaining entities - decode_entities($s); - - return $s; +sub debug_print { + if ($opt->{debug}) { + my ($msg) = shift; + print STDERR encode_and_trim( $msg ) . "\n"; } } -# Returns a TreeBuilder instance for a given URL. The URL is retrieved -# via get_nice(), decoded into a Perl string, processed to remove HTML -# entities and then parsed into a HTML::TreeBuilder object -# -sub get_tree { - my $url = shift; - my $content; - my $nbretry = 0; - while ( (not defined($content = get_nice($url))) || (length($content) == 0) ) { - ++$nbretry; - return undef if $nbretry > $MAX_RETRY; - debug_print("*** Retrying URL: '" . $url - . "' (attempt " . $nbretry . " of " . $MAX_RETRY . ") ***"); - } - $content = decode_utf8($content); - $content = tidy_html($content); - my $t = new HTML::TreeBuilder; - $t->parse($content) or die "Cannot parse content of Tree\n"; - $t->eof; - return $t; -} +__END__ -# Return a DateTime object representing 'now' -# -sub get_date_today { - return DateTime->now( time_zone => 'Europe/Paris' ); -} +=pod -# Return a DateTime object represent 'now' + $offset days -# -sub get_date_today_with_offset { - my $offset = DateTime::Duration->new( days => shift ); - return get_date_today() + $offset; -} +=encoding utf8 + +=head1 NAME + +tv_grab_fr - Grab TV listings for France (Télé Star). + +=head1 SYNOPSIS + + To configure: + tv_grab_fr --configure [--config-file FILE] [--gui OPTION] + + To list available channels: + tv_grab_fr --list-channels + + To grab listings: + tv_grab_fr [--config-file FILE] [--output FILE] + [--days N] [--offset N] [--slow] + [--quiet | --debug] + + To show capabilities: + tv_grab_fr --capabilities + + To show version: + tv_grab_fr --version + + To display help: + tv_grab_fr --help + +=head1 DESCRIPTION + +Output TV listings for many channels available in France (Orange, +Free, cable/ADSL/satellite, Canal+ Sat). The data comes from +Télé Star (telestar.fr). The default is to grab 14 days. + +B<--configure> Choose which bouquet/channels to grab listings data for. + +B<--list-channels> List available channels. + +B<--config-file FILE> Use FILE as config file instead of the default config +file. This allows for different config files for different applications. + +B<--gui OPTION> Use this option to enable a graphical interface to be used. +OPTION may be 'Tk', or left blank for the best available choice. +Additional allowed values of OPTION are 'Term' for normal terminal output +(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar. + +B<--output FILE> Write to FILE rather than standard output. + +B<--days N> Grab N days (default: 14) starting from today. + +B<--offset N> Start grabbing N days from today, rather than starting +today. + +B<--slow> Download additional information (e.g. longer description, +cast details) for each programme, where available. This option +significantly slows down the grabber and is disabled by default. + +B<--quiet> Suppress the progress messages normally written to standard +error. + +B<--debug> Provide additional debugging messages during processing. + +B<--capabilities> Show which capabilities the grabber supports. For more +information, see L. + +B<--version> Show the version of the grabber. + +B<--help> Print a help message and exit. + +=head1 SEE ALSO + +L + +=head1 AUTHOR + +The current tv_grab_fr script was rewritten by Nick Morrott, +knowledgejunkie at gmail dot com, to support the new telestar.fr site. + +=cut -# Return 'now' as dd-mm-yyyy string -sub get_date_ddmmyyyy { - return get_date_today->strftime("%d-%m-%Y"); -} diff -Nru xmltv-0.6.1/grab/fr_kazer/test.conf xmltv-0.6.3/grab/fr_kazer/test.conf --- xmltv-0.6.1/grab/fr_kazer/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fr_kazer/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -userhash=sfv12sgydswv7 -channel=FRA2.kazer.org -channel=FRA3.kazer.org -channel!TF11.kazer.org -channel!TV51.kazer.org diff -Nru xmltv-0.6.1/grab/fr_kazer/tv_grab_fr_kazer xmltv-0.6.3/grab/fr_kazer/tv_grab_fr_kazer --- xmltv-0.6.1/grab/fr_kazer/tv_grab_fr_kazer 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/fr_kazer/tv_grab_fr_kazer 1970-01-01 00:00:00.000000000 +0000 @@ -1,316 +0,0 @@ -#!/usr/bin/perl -w - -=pod - -=head1 NAME - -tv_grab_fr_kazer - Grab TV listings from Kazer in France. - -=head1 SYNOPSIS - -tv_grab_fr_kazer --help - -tv_grab_fr_kazer --configure [--config-file FILE] - -tv_grab_fr_kazer [--config-file FILE] - [--days N] [--offset N] [--channel xmltvid,xmltvid,...] - [--output FILE] [--quiet] [--debug] - -tv_grab_fr_kazer --list-channels [--config-file FILE] - [--output FILE] [--quiet] [--debug] - - -=head1 DESCRIPTION - -Output TV and listings in XMLTV format for many stations -available in France. This program consumes the EPG service offering -from Kazer at L. -See TODO INSERTLINKHERE -for their terms of service. (automatic translation suggests it's free -for personal use, verification appreciated) - -First you must register an account at kazer.org and choose which stations -you want to receive. - -Then run B to setup the grabber with your -userhash and maybe filter the channels. (in case you feed multiple -consumers with data from one account, like using different configurations -for tv cable and iptv channels) - -After these two steps running B with no arguments will -get a listings for the stations you chose for all available days -including today. - -ATTENTION: The downloaded data is cached for one hour, so if you add channels -to your account it can take up to one hour until they are returned from the -grabber. You can remove the downloaded guide to force an immediate refresh -by deleting the guide from the supplementry file cache. On Unix like system -it is by default at ~/.xmltv/supplement/tvguide.zip?u=. - -=head1 OPTIONS - -B<--configure> Prompt for which stations to download and write the -configuration file. - -Note that due to the format of the source data, tv_grab_fr_kazer always -downloads data for all channels that have been selected on kazer.org. -Removing channels from the configuration in order to speed up downloads or -reduce data transfer will therefore not work. - -B<--config-file FILE> Set the name of the configuration file, the -default is B<~/.xmltv/tv_grab_fr_kazer.conf>. This is the file written by -B<--configure> and read when grabbing. - -B<--output FILE> When grabbing, write output to FILE rather than -standard output. - -B<--days N> When grabbing, grab N days rather than everything available. - -B<--offset N> Start grabbing at today + N days. - -Note that due to the format of the source data, tv_grab_fr_kazer always -downloads data for all available days and then filters for days specified -with --days and --offset. Specifying --days and/or --offset in order to -speed up downloads or reduce data transfer will therefore not work. - -B<--quiet> Only print error-messages on STDERR. - -B<--debug> Provide more information on progress to stderr to help in -debugging. - -B<--list-channels> Output a list of all channels that data is available - for. The list is in xmltv-format. - -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L - -B<--version> Show the version of the grabber. - -B<--help> Print a help message and exit. - -=head1 ERROR HANDLING - -If the grabber fails to download data from kazer.org, it will print an -errormessage to STDERR and then exit with a status code of 1 to indicate -that the data is missing. - -=head1 ENVIRONMENT VARIABLES - -The environment variable HOME can be set to change where configuration -files are stored. All configuration is stored in $HOME/.xmltv/. On Windows, -it might be necessary to set HOME to a path without spaces in it. - -=head1 CREDITS - -Grabber written by Karl Dietz, dekarl -at- users -dot- sourceforge -dot- net -as a an example on how to wrap a file download into a well formed grabber. -This documentation copied from tv_grab_cz by Mattias Holmlund, -This documentation copied from tv_grab_uk by Ed Avis, -ed -at- membled -dot- com. Original grabber by Jiri Kaderavek, -jiri -dot- kaderavek -at- webstep -dot- net with modifications by -Petr Stehlik, pstehlik -at- sophics -dot- cz. - -Data provided via web service from kazer.org. Check their terms of usage! - -=head1 BUGS - -None known. - -=cut - -use strict; -use DateTime; -use Encode; # used to convert 'perl strings' into 'utf-8 strings' -use IO::Uncompress::Unzip qw/unzip/; -use XML::LibXML; - -use XMLTV; -use XMLTV::Configure::Writer; -use XMLTV::Options qw/ParseOptions/; -use XMLTV::Supplement qw/GetSupplement SetSupplementRoot/; - -# TODO verify if we need to switch between kazer.org and xmltv.org to avoid messing with the downloaded xmltv.dtd -SetSupplementRoot( 'http://www.kazer.org/' ); - -# kazer.org holds 7-8 days currently, likely to raise to 14 in the future. -my $maxdays = 1+14; # data source is limited to n days (including today) - -my( $opt, $conf ) = ParseOptions( { - grabber_name => "tv_grab_fr_kazer", - capabilities => [qw/apiconfig baseline manualconfig preferredmethod/], - listchannels_sub => \&list_channels, - stage_sub => \&config_stage, - version => "$XMLTV::VERSION", - description => "France (Kazer)", - # prefer 'allatonce' as we download one big zip file and filter it afterwards - preferredmethod => 'allatonce', - defaults => { days => $maxdays, offset => 0, quiet => 0, debug => 0 }, -} ); - -if (not defined( $conf->{'userhash'} )) { - print STDERR "No userhash selected in configfile " . - $opt->{'config-file'} . "\n" . - "Please run the grabber with --configure.\n"; - exit 1; -} - -if (not defined( $conf->{'channel'} )) { - print STDERR "No channels selected in configfile " . - $opt->{'config-file'} . "\n" . - "Please run the grabber with --configure.\n"; - exit 1; -} - -# hash of channels for the channel/programme callback -my %channels = map { $_ => $_ } @{$conf->{channel}}; - -# limit to maxdays in the future -if ($opt->{offset} + $opt->{days} > $maxdays) { - $opt->{days} = $maxdays - $opt->{offset}; -} - -if ($opt->{days} < 1) { - $opt->{days} = 0; -} - -# Get the actual data and print it to stdout. -my $is_success=1; - -my $epgsource; -my $epgsourcezip; -if ($opt->{days} > 0) { - $epgsourcezip = GetSupplement( undef, 'tvguide.zip?u=' . $conf->{'userhash'}->[0] ); -} else { - if( !$opt->{quiet} ) { - print( STDERR "no data available for the requested time period\n" ); - } - $is_success = 0; -} - -unzip \$epgsourcezip => \$epgsource; - -# TODO some fixups, they have been reported to the site and can be remove once they are not needed anymore -# FIXME XMLTV::parse_callback doesn't seem to read role="" -#$epgsource =~ s|(.*?) \((.*?)\)|$1|g; - - -# === setup writer === -my %w_args = ( - cutoff => '000000', - days => $opt->{days}, - encoding => 'UTF-8', - offset => $opt->{offset} -); - -my $writer = new XMLTV::Writer( %w_args ); -# as suggested on http://wiki.xmltv.org/index.php/Supplementary_Files -# FIXME does not work because the XMLTV::Writer forces a SYSTEM of xmltv.id on us -#$writer->doctype( 'tv', undef, 'http://supplement.xmltv.org/xmltv.dtd' ); -$writer->start({ - 'generator-info-name' => "XMLTV/$XMLTV::VERSION", - 'generator-info-url' => 'http://www.xmltv.org/', - 'source-info-name' => 'KaZeR\'s XMLTV', - 'source-info-url' => 'http://www.kazer.org/', -}); - - -# === the callbacks === -my $encoding; -sub encoding_cb( $ ) { $encoding = shift } - -my $credits; -sub credits_cb( $ ) { $credits = shift } - -# The callback for each channel gets filtered by the hash of configured channels. -sub channel_cb( $ ) { - my $c = shift; - - if( defined( $channels{$c->{id}} ) ) { - $writer->write_channel($c); - } -} - -# The callback for each programme. We know that channels are -# always read before programmes, so the %channels hash will be -# fully populated. -# -# It just filters the programmes by configured channel. The filtering by time is done by the Writer itself. -# -sub programme_cb( $ ) { - my $p = shift; - - if( defined( $channels{$p->{channel}} ) ) { - $writer->write_programme($p); - } -} - - -# Let's go. -XMLTV::parse_callback($epgsource, \&encoding_cb, \&credits_cb, - \&channel_cb, \&programme_cb); - - -$writer->end(); - -if( $is_success ) { - exit 0; -} else { - exit 1; -} - -sub config_stage -{ - my( $stage, $conf ) = @_; - - # Sample stage_sub that only needs a single stage. - - die "Unknown stage $stage" if $stage ne "start"; - - my $result; - my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, - encoding => 'utf-8' ); - $writer->start( { grabber => 'tv_grab_fr_kazer' } ); - - # TODO add french strings - $writer->write_string( { - id => 'userhash', - title => [ [ 'Userhash', 'en' ] ], - description => [ [ 'Your userhash at kazer.org. It is shown on ' . - 'http://www.kazer.org/my-channels.html together with your ' . - 'selection of channels.', 'en' ], - [ encode('utf-8', "Votre userhash pour kazer.org. Il est indiqu\xe9 sur " . - 'http://www.kazer.org/my-channels.html au dessus du ' . - "s\xe9lecteur de cha\xeenes." ), 'fr' ] ], - default => '', - } ); - - $writer->end( 'select-channels' ); - - return $result; -} - -sub list_channels -{ - my( $conf, $opt ) = @_; - - # Return a string containing an xmltv-document with -elements - # for all available channels. - - my $channellist = GetSupplement( undef, 'tvguide.zip?u=' . $conf->{'userhash'}->[0] ); - my $xml; - unzip \$channellist => \$xml; - - my $parser=XML::LibXML->new(); - my $input=$parser->parse_string( $xml )->getDocumentElement(); - - my $output=XML::LibXML::Document->new( '1.0', 'utf-8' ); - my $root=XML::LibXML::Element->new( 'tv' ); - $output->setDocumentElement( $root ); - - foreach my $channel( $input->getElementsByTagName( 'channel') ) { - $root->appendChild( $channel ); - } - - return $output->toString(); -} - diff -Nru xmltv-0.6.1/grab/huro/tv_grab_huro.in xmltv-0.6.3/grab/huro/tv_grab_huro.in --- xmltv-0.6.1/grab/huro/tv_grab_huro.in 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/huro/tv_grab_huro.in 2020-09-07 15:02:53.000000000 +0000 @@ -300,8 +300,8 @@ #------------------------------------------------------------------------------- sub xhead() { my $d = &domain; - return { 'source-info-url' => "http://www.$d/", - 'source-data-url' => "http://www.$d/tv/", + return { 'source-info-url' => "https://www.$d/", + 'source-data-url' => "https://www.$d/tv/", 'generator-info-name' => 'XMLTV', 'generator-info-url' => 'http://xmltv.org/', }; @@ -751,7 +751,7 @@ $to_date += ONE_DAY * $days_to_request; $ch_port_id =~ s/^0+//; my $d = domain(); - my $urlfmt = "http://" . $d . (($COUNTRY eq 'hu') ? "/tvapi?channel_id=tvchannel-" : "/pls/w/tv_api.event_list?i_channel_id=").$ch_port_id. + my $urlfmt = "https://" . $d . (($COUNTRY eq 'hu') ? "/tvapi?channel_id=tvchannel-" : "/pls/w/tv_api.event_list?i_channel_id=").$ch_port_id. "&i_datetime_from=%s&i_datetime_to=".$to_date->strftime('%Y-%m-%d'); my $url = "$urlfmt"; local $SIG{__WARN__} = sub { @@ -895,7 +895,7 @@ $program{desc} =~ s/Megvásárolható (DVD[ ]?-n|VHS[ ]?-en)//g; } if ($prog_data->{'film_url'}) { - my @url = ((($COUNTRY eq 'hu') ? 'http://'.domain() : '').$prog_data->{'film_url'}); + my @url = ((($COUNTRY eq 'hu') ? 'https://'.domain() : '').$prog_data->{'film_url'}); $program{infourl} = \@url; } @@ -1156,7 +1156,7 @@ my $d = domain(); my $bar = new XMLTV::ProgressBar('getting list of channels', 1) if not $opt_quiet; - my $url="http://www.$d/pls/tv/".($COUNTRY eq 'hu' || $COUNTRY eq 'ro' ? 'old' : '')."tv.prog"; # bug #501 + my $url="https://www.$d/pls/tv/".($COUNTRY eq 'hu' || $COUNTRY eq 'ro' ? 'old' : '')."tv.prog"; # bug #501 worker("base-downloading"); t "fetching $url..."; @@ -1226,7 +1226,7 @@ my $d = domain(); my $bar = new XMLTV::ProgressBar('getting list of channels', 1) if not $opt_quiet; - my $url="http://www.$d/".(($COUNTRY eq 'hu') ? "tvapi/init" : "pls/w/tv_api.init?i_page_id=1"); + my $url="https://www.$d/".(($COUNTRY eq 'hu') ? "tvapi/init" : "pls/w/tv_api.init?i_page_id=1"); worker("base-downloading"); t "fetching $url..."; @@ -1241,7 +1241,7 @@ my $channel_id = $ch->{'id'}; $channel_id =~ s/^[^0-9]+//; # 'tvchannel-N' -> 'N' $channel_id = sprintf("%03d", $channel_id); # 'N' -> '00N' - my @urls = (($COUNTRY eq 'hu') ? 'http://'.domain() : '').$ch->{'link'}; + my @urls = (($COUNTRY eq 'hu') ? 'https://'.domain() : '').$ch->{'link'}; my %channel = ( 'display-name' => [ [ $ch->{'name'}, $COUNTRY ] ], 'id' => "$channel_id.$d", @@ -1300,7 +1300,7 @@ # add port.hu/port.ro base url only if url is not contains the "://" uri separator if (! ($url =~ "://")) { - $url = "http://www.$d" . $url; + $url = "https://www.$d" . $url; } # no info, so don't add it to anywhere @@ -1732,7 +1732,7 @@ # add port.hu/port.ro base url only if url is not contains the "://" uri separator if (! ($url =~ "://")) { - $url = "http://www.$d" . $url; + $url = "https://www.$d" . $url; } # no info, so don't add it to anywhere @@ -1864,7 +1864,7 @@ return unless ($opt_icons || $opt_local_icons); my $channelid = shift; - my $fetchurl = "http://www." . domain() . "/tv/kep_ado/al_".(int(${channelid}) % 10000).".gif"; + my $fetchurl = "https://www." . domain() . "/tv/kep_ado/al_".(int(${channelid}) % 10000).".gif"; my ($file, $iconurl); # that $fetchurl no longer works for RO, so... @@ -1874,7 +1874,7 @@ if (!defined $image) { # image url not valid, so we must get it from the programmes page. Ideally we would do that during the main grab but this is a Q&D fix # and I don't want to change too much of this code - my $url = "http://www." . domain() . "/pls/w/".($COUNTRY eq 'hu' || $COUNTRY eq 'ro' ? 'old' : '')."tv.channel?i_ch=".$channelid."&i_date=".UnixDate('today','%Y-%m-%d')."&i_where=1"; # bug #501 + my $url = "https://www." . domain() . "/pls/w/".($COUNTRY eq 'hu' || $COUNTRY eq 'ro' ? 'old' : '')."tv.channel?i_ch=".$channelid."&i_date=".UnixDate('today','%Y-%m-%d')."&i_where=1"; # bug #501 my $data=get_nice($url); my $tree = HTML::TreeBuilder->new_from_content($data) or @@ -1887,7 +1887,7 @@ if (my $imgdiv = $container->look_down("_tag" => "div", "style" => qr/float\s*:\s*left/, sub { my $imgtag = $_[0]->look_down('_tag' => 'img'); return 0 unless $imgtag; - return $imgtag->attr('src') =~ m/http:\/\/media/; + return $imgtag->attr('src') =~ m/https:\/\/media/; } )) { $fetchurl = $imgdiv->look_down('_tag' => 'img')->attr('src'); @@ -1963,7 +1963,7 @@ my $chdata; # two sprintf parameters: first: channel_id,, second how many days grabbed - my $churlfmt = "http://www." . domain() . + my $churlfmt = "https://www." . domain() . "/pls/tv/".($COUNTRY eq 'hu' || $COUNTRY eq 'ro' ? 'old' : '')."tv.channel?i_ch=%d&" . "i_days=1&i_xday=%d&i_where=1"; # bug #501 diff -Nru xmltv-0.6.1/grab/in_toi/test.conf xmltv-0.6.3/grab/in_toi/test.conf --- xmltv-0.6.1/grab/in_toi/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/in_toi/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -channel=STAR PLUS -channel=Zee TV -channel=BBC World News diff -Nru xmltv-0.6.1/grab/in_toi/tv_grab_in_toi xmltv-0.6.3/grab/in_toi/tv_grab_in_toi --- xmltv-0.6.1/grab/in_toi/tv_grab_in_toi 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/in_toi/tv_grab_in_toi 1970-01-01 00:00:00.000000000 +0000 @@ -1,300 +0,0 @@ -#!/usr/bin/perl -w - -eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}' - if 0; # not running under some shell - -=pod - -=head1 NAME - -tv_grab_in_toi - Grab TV listings for India from Times Of India website - -=head1 SYNOPSIS - -tv_grab_in_toi --help - -tv_grab_in_toi --configure [--config-file FILE] - -tv_grab_in_toi [--config-file FILE] - [--days N] [--offset N] - [--output FILE] [--quiet] - -tv_grab_in_toi --list-channels [--config-file FILE] - [--output FILE] [--quiet] - - -=head1 DESCRIPTION - -Outputs TV listings in XMLTV format for stations available in -India. - -Then running B with no arguments will get listings for -all the channels. - -=head1 OPTIONS - -B<--configure> Prompt for which stations to download and write the -configuration file. - -B<--config-file FILE> Set the name of the configuration file, the -default is B<~/.xmltv/tv_grab_in_toi.conf>. This is the file written by -B<--configure> and read when grabbing. - -B<--output FILE> When grabbing, write output to FILE rather than -standard output. - -B<--days N> When grabbing, grab N days rather than the default. - -B<--offset N> Start grabbing at today + N days. Also supports negative offset for past listings. - -B<--quiet> Only print error-messages on STDERR. - -B<--debug> Provide more information on progress to stderr to help in -debugging. - -B<--list-channels> Output a list of all channels that data is available - for. The list is in xmltv-format. - -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L - -B<--version> Show the version of the grabber. - -B<--help> Print a help message and exit. - -=head1 ERROR HANDLING - -If the grabber fails to download data, it will print an -errormessage to STDERR and then exit with a status code of 1 to indicate -that the data is missing. - -=head1 CREDITS - -Grabber written by Anand Tamariya, atamariya@gmail.com -This documentation copied from tv_grab_cz, -This code modified from tv_grab_cz, by Mattias Holmlund, mattias -at- holmlund -dot- se. -L - -=head1 BUGS - -None known. - -=cut - -use strict; -use warnings; -use XMLTV; -use XMLTV::Version "$XMLTV::VERSION"; -use XMLTV::Configure::Writer; -use XMLTV::Options qw/ParseOptions/; -use XML::LibXML; -use LWP::Simple; -use Date::Calc qw(Add_Delta_Days); -use URI::Escape qw(uri_escape); -use JSON; -use Data::Dumper; - -# config vars -my $webroot = "http://timesofindia.indiatimes.com/"; -my $loginURL = "http://timesofindia.indiatimes.com/"; - -my( $opt, $conf ) = ParseOptions( { - grabber_name => "tv_grab_in_toi", - capabilities => [qw/baseline manualconfig apiconfig/], - stage_sub => \&config_stage, - listchannels_sub => \&list_channels, - version => "$XMLTV::VERSION", - description => "India (timesofindia.indiatimes.com)", -} ); - -# make URL -my @channels = @{$conf->{channel}}; - -my $data; -$data->[0] = 'UTF-8'; -$data->[1] = { - 'source-info-url' => 'http://timesofindia.indiatimes.com/', - 'source-info-name' => 'Times of India', - 'generator-info-name' => 'tv_grab_in_toi', - 'generator-info-url' => 'http://wiki.xmltv.org/index.php/XMLTVProject' - }; - - -# channels are retrieved by display name not id -# -while (my @next_n = splice @channels, 0, 10) { - my $channels_str = join ",", @next_n; - my $url = buildURL( 'get_schedule', $conf, $opt, $channels_str ); - - # fetch data - my $res = get( $url ); - #print $res; - transform_program($res); -} - -XMLTV::write_data($data); - -exit(0); - - -#========================================================================================= - -sub config_stage -{ - my( $stage, $conf ) = @_; - my $result; - - my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, encoding => 'utf-8' ); - - if( $stage eq 'start' ) { - $writer->start( { grabber => 'tv_grab_in_toi' } ); - $writer->end( 'select-channels' ); - - } else { - die "Unknown stage $stage"; - } - - return $result; -} - -# Return a string containing an xml-document with -elements -# for all available channels -sub list_channels -{ - # $opt hold command line parameters, if any - my( $conf, $opt ) = @_; - - my $url = buildURL( 'get_channels', $conf, $opt ); - - # fetch list of channels - my $channels = get( $url ); - my $data = transform_channel( $channels ); - - # return XML string - return $data; -} - -# Transform comma separated string to XMLTV channel list in string form -sub transform_channel -{ - my ( $str ) = @_; - - # we don't get JSON but a simple comma-separated list of channel names! - my @array = split( ",", $str); - - my $result; - my $writer = new XMLTV::Writer(OUTPUT => \$result); - $writer->start({'generator-info-name' => 'tv_grab_in_toi'}); - - # Note this breaks RFC2838 but there's no easy way to get the list of channel ids - # - foreach my $name (@array) { - my %channel = ( - 'id' => "$name", - 'display-name' => [ [ "$name", "en" ] ] - ); - $writer->write_channel( \%channel ); - } - $writer->end(); - #printf $result; - - return $result; -} - -# Transform schedule JSON to XMLTV in $data -sub transform_program -{ - my ( $str ) = @_; - my $ptr = decode_json $str; - $ptr = $ptr->{"ScheduleGrid"}->{"channel"}; - - my $tz = "+0530"; # IST - my $lang = "en"; - my ($encoding, $credits, $ch, $progs) = @$data; - foreach my $c (@$ptr) { - my $channeldisplayname = $c->{"channeldisplayname"}; - $channeldisplayname =~ s/\s/-/g; # spaces not allowed by RFC2838 (note this still breaks validation as it's not a valid dns-identifier) - if (! exists($ch->{$c->{"channelid"}})) { - $ch->{$c->{"channelid"}} = { - # we should use channelid but that's not how they're listed in the config file, - # and we should be consistent! - # id => $c->{"channelid"}, - 'id' => $channeldisplayname, - 'icon' => [ { src => $c->{"channellogourl"}} ], - 'display-name' => [ [ $c->{"display-name"}, $lang ] ] - }; - } - my $program = $c->{"programme"}; - foreach my $p (@$program) { - - my %prog = ( - #channel => $c->{"channelid"}, - channel => $channeldisplayname, - start => "$p->{'start'} $tz", - stop => "$p->{'stop'} $tz", - category => [ [$p->{"subgenre"}, $lang] ], - icon => [ { src => $p->{"programmeurl"}} ], - date => $p->{"date"}, - title => [ [ $p->{"title"}, $lang ] ] - ); - push @$progs, \%prog; - } - } - - $data->[2] = $ch; - $data->[3] = $progs; - #print Dumper $ch; -} - -# build API URL for a given action -sub buildURL -{ - my( $action, $conf, $opt, $channels ) = @_; - my $url; - - if ($action eq "get_channels") { - $url = $webroot . "tvschannellist.cms?genrename=all"; - return $url; - - } elsif ($action eq "get_schedule") { - $channels =~ s/\ /%20/g ; - $url = $webroot . "tvschedulejson.cms?channellist=" . $channels; - - } else { - die "Invalid action\n"; - } - - if ( !exists($opt->{quiet}) || $opt->{quiet} == 0 ) { - if ( $opt->{days} > 1 ) { - die "EPG only allows fetch of one day at a time\n"; - } - } - - my ($sec,$min,$hour,$day,$month,$year,$wday,$yday,$isdst) = localtime(); - my $datestring; - $year += 1900; - $month++; - - if( $opt->{offset} > 0 ) { - # append offset - ($year, $month, $day) = Add_Delta_Days($year, $month, $day, $opt->{offset}); - } - $datestring = sprintf "%04d%02d%02d0000", ($year, $month, $day); - $url .= "&fromdatetime=" . $datestring; - - if( $opt->{days} > 0 ) { - $opt->{days} = 1; # can only fetch one day at a time - # append days - ($year, $month, $day) = Add_Delta_Days($year, $month, $day, $opt->{days}); - $datestring = sprintf "%04d%02d%02d0000", ($year, $month, $day); - $url .= "&todatetime=" . $datestring; - } - - #print STDERR $url; - # e.g. http://timesofindia.indiatimes.com/tvschedulejson.cms?channellist=Zee%20TV&fromdatetime=201802240000&todatetime=201802250000 - - return $url; - -} - -1; diff -Nru xmltv-0.6.1/grab/it/tv_grab_it.in xmltv-0.6.3/grab/it/tv_grab_it.in --- xmltv-0.6.1/grab/it/tv_grab_it.in 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/it/tv_grab_it.in 2020-09-07 15:02:53.000000000 +0000 @@ -2429,6 +2429,8 @@ #split and parse the lines my @lines = split /\n/, $content; + my $day = $grabdate; + my $last_start2 = 0; foreach my $line (@lines) { next unless ($line=~/^=0) { #they can work as decimals, 0.32 < 23.44 - push @prog_to_check, [$title, $tomorrowdate." ".$start]; + my $start2 = $start =~ s/:/\./r; # change start2 to decimal + # detect date change (page contains shows from 06:00 to 06:00 next day + if ($start2 < $last_start2) { + $day = $tomorrowdate; } - else { - push @prog_to_check, [tidy($title), $grabdate." ".$start]; - } + # sometimes two programs have the same start2 (like meteo), keep only the last one + if ($start2 == $last_start2) { + pop(@prog_to_check); + } + + push @prog_to_check, [tidy($title), $day." ".$start]; + $last_start2 = $start2; } + # ignore last prog to avoid duplicate with next day + pop(@prog_to_check); diff -Nru xmltv-0.6.1/grab/na_dd/tv_grab_na_dd.in xmltv-0.6.3/grab/na_dd/tv_grab_na_dd.in --- xmltv-0.6.1/grab/na_dd/tv_grab_na_dd.in 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/na_dd/tv_grab_na_dd.in 2020-09-07 15:02:53.000000000 +0000 @@ -705,6 +705,7 @@ } my $dd_service='http://dd.schedulesdirect.org/tech/tmsdatadirect/schedulesdirect/tvDataDelivery.wsdl'; + $dd_service='https://dd.schedulesdirect.org/tech/tmsdatadirect/schedulesdirect/ssl_tvDataDelivery.wsdl'; $dd_service=$ENV{DD_SERVICE} if exists $ENV{DD_SERVICE}; # used for testing my $proxy='http://localhost/'; diff -Nru xmltv-0.6.1/grab/na_dtv/test.conf xmltv-0.6.3/grab/na_dtv/test.conf --- xmltv-0.6.1/grab/na_dtv/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/na_dtv/test.conf 2020-09-07 15:02:53.000000000 +0000 @@ -438,10 +438,10 @@ channel!0005.directv.com channel!0500.directv.com channel=0501.directv.com -channel!0502.directv.com -channel!0503.directv.com -channel!0504.directv.com -channel!0505.directv.com +channel=0502.directv.com +channel=0503.directv.com +channel=0504.directv.com +channel=0505.directv.com channel!0506.directv.com channel!0507.directv.com channel!0508.directv.com diff -Nru xmltv-0.6.1/grab/na_dtv/tv_grab_na_dtv xmltv-0.6.3/grab/na_dtv/tv_grab_na_dtv --- xmltv-0.6.1/grab/na_dtv/tv_grab_na_dtv 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/na_dtv/tv_grab_na_dtv 2020-09-07 15:02:53.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl =pod @@ -12,12 +12,13 @@ tv_grab_na_dtv --configure [--config-file FILE] -tv_grab_na_dtv [--config-file FILE] - [--days N] [--offset N] [--processes N] - [--output FILE] [--quiet] [--debug] +tv_grab_na_dtv [--config-file FILE] [--output FILE] + [--days N] [--offset N] [--processes N] + [--timeout N] [--agent STRING] + [--quiet | --debug] tv_grab_na_dtv --list-channels [--config-file FILE] - [--output FILE] [--quiet] [--debug] + [--output FILE] [--quiet | --debug] =head1 DESCRIPTION @@ -45,6 +46,10 @@ B<--offset N> Start grabbing at today + N days. +B<--timeout N> Set request timeout in seconds. Default is 240 seconds. + +B<--agent STRING> Specify browser agent string, as provided to remote server. + B<--processes N> Number of processes to run to fetch program details. 8 is a good number to try. You could try more with plenty of CPU and bandwidth. More processes will reduce the time it takes to fetch your @@ -58,8 +63,8 @@ B<--debug> Provide more information on progress to stderr to help in debugging. -B<--list-channels> Output a list of all channels that data is available - for. The list is in xmltv-format. +B<--list-channels> Output a list of all channels that data is available +for. The list is in xmltv-format. B<--capabilities> Show which capabilities the grabber supports. @@ -86,7 +91,7 @@ Grabber written Rod Roark (http://www.sunsetsystems.com/), Modified by Adam Lewandowski (adam@alewando.com) (January 2011, October 2014) - to account for DirecTV site/API changes. +to account for DirecTV site/API changes. =head1 BUGS @@ -99,6 +104,7 @@ =cut use strict; +use warnings; use XMLTV; use XMLTV::Configure::Writer; use XMLTV::Options qw/ParseOptions/; @@ -168,6 +174,10 @@ # chNum, chCall, chName, chLogoUrl, chId my %ch = (); +# Create running time for debugging messages +my $start_runtime = time(); + +# ###################################################################### # Main logic starts here # ###################################################################### @@ -294,7 +304,7 @@ ###################################################################### sub getBrowser { - my ($conf) = @_; + my ($opt, $conf) = @_; my $ua = LWP::UserAgent->new; @@ -305,11 +315,8 @@ $cookies->set_cookie(0, 'dtve-prospect-zip', "$zip", '/', 'www.directv.com'); - # Define user agent type - $ua->agent('Mozilla/5.0 (Linux) XmlTv'); - - # Define timouts - $ua->timeout(240); + $ua->timeout($opt->{timeout}); + $ua->agent($opt->{agent}); # Use proxy if set in http_proxy etc. $ua->proxy( [ 'http', 'https' ], $conf->{proxy}->[0] ) @@ -358,7 +365,7 @@ } # For sorting %ch by its (channel number) key: -sub numerically { $a cmp $b } +sub numerically { $a <=> $b } # This is what the main process does first. Variables in here will # go nicely out of scope before the child processes are started. @@ -372,8 +379,8 @@ listchannels_sub => \&list_channels, version => "$XMLTV::VERSION", description => "North America using www.directv.com", - extra_options => [qw/procs=i processes=i/], # allow 'procs' as a synonym for 'processes' - defaults => {'procs'=>0, 'processes'=>'0'} + extra_options => [qw/procs=i processes=i timeout=i agent=s/], # allow 'procs' as a synonym for 'processes' + defaults => {'procs' => 0, 'processes' => 0, 'timeout' => 240, 'agent' => "xmltv/$XMLTV::VERSION"}, } ); @@ -389,7 +396,7 @@ $timeZone = $conf->{timezone}[0]; $timeZone = "America/New_York" if !$timeZone; # Default to EST - $browser = getBrowser($conf); + $browser = getBrowser($opt, $conf); # Populate %ch hash &scrape_channel_list( $browser, $conf->{zip}[0], $conf->{channel}, \%ch ); @@ -450,10 +457,16 @@ sub scrape_channel_list { my ( $browser, $zip, $channels, $ch ) = @_; - print STDERR "Getting channel list\n" if ($DEBUG); - my $resp = $browser->get($CHANNEL_LIST_URL); - my $json = $resp->content(); - my $data = decode_json $json; + print STDERR "Requesting channel list using url: '$CHANNEL_LIST_URL' \n" if ($DEBUG); + + my $can_accept = HTTP::Message::decodable; + my $resp = $browser->get($CHANNEL_LIST_URL, 'Accept-Encoding' => $can_accept, 'Accept-Language' => "en-US,en;q=0.9",); + + #my $json = $resp->decoded_content(); + my $data = eval { decode_json( $resp->decoded_content() ) }; + if ($@) { + print STDERR "Decoding JSON response failed. \n" if ($VERBOSE); + } # Check status code if ( !$data->{success} ) { @@ -542,7 +555,7 @@ $VERBOSE = !$opt->{quiet}; - my $browser = getBrowser($conf); + my $browser = getBrowser($opt, $conf); &scrape_channel_list( $browser, $conf->{zip}[0], $conf->{channel}, \%ch ); my $xml = $XML_PRELUDE; @@ -667,9 +680,9 @@ # Check that the start date is on the requested day (after adjusting to the specified timezone) sub checkStartDate { - my ($day, $startDt) = @_; - #my $cmpDay = $startDt->ymd; - return $startDt->ymd eq $day->ymd; + my ($day, $startDt) = @_; + #my $cmpDay = $startDt->ymd; + return $startDt->ymd eq $day->ymd; } # This generates XML for the designated channel on the designated day @@ -687,11 +700,22 @@ my $url = URI->new($SCHEDULE_URL); #TODO: Include chIds parameter (comma-sep list of channel IDs corresponding to channel numbers) $url->query_form('channels' => $shortNum, 'startTime' => $starttime, 'hours' => $blockduration); - my $resp = $browser->get($url); - my $json = $resp->content(); + + if ($DEBUG) { + my $exec_duration = time() - $start_runtime; + print STDERR "$exec_duration - Requesting channel's daily data using url: '$url' \n"; + } + + my $can_accept = HTTP::Message::decodable; + my $resp = $browser->get($url, 'Accept-Encoding' => $can_accept, 'Accept-Language' => "en-US,en;q=0.9",); + + #my $json = $resp->decoded_content(); # Parse JSON - my $data = decode_json $json; + my $data = eval { decode_json( $resp->decoded_content() ) }; + if ($@) { + print STDERR "Decoding JSON response failed. \n" if ($VERBOSE); + } # Check status code if ( $data->{errors} ) { @@ -710,13 +734,13 @@ if( ref($data->{schedule}) eq 'ARRAY' && @{ $data->{schedule}} ) { my @schedules = @{ @{ $data->{schedule}}[0]->{schedules} }; foreach my $prog (@schedules) { - # Skip if program does not start on the target date (ie: started the previous day) - my $startDate = parseDate($prog->{airTime}); - $prog->{startDt} = $startDate; - if (!checkStartDate(parseDate($day), parseDate($prog->{airTime}) )) { - print STDERR "Skipping program $prog->{programID} because it doesn't start on $day: $prog->{airTime}\n" if ($DEBUG); - next; - } + # Skip if program does not start on the target date (ie: started the previous day) + my $startDate = parseDate($prog->{airTime}); + $prog->{startDt} = $startDate; + if (!checkStartDate(parseDate($day), parseDate($prog->{airTime}) )) { + print STDERR "Skipping program $prog->{programID} because it doesn't start on $day: $prog->{airTime}\n" if ($DEBUG); + next; + } # Get program details $output .= &scrape_program_details( $browser, $channel_number, $prog ); @@ -736,6 +760,8 @@ my $length = $program_data->{duration}; my $hd = $program_data->{hd}; + my $exec_duration; + return "" if $program_id eq "-1"; # Append '-1' to channel number if this is an HD broadcast on a dual-numbered channel @@ -753,16 +779,30 @@ # Get program details page my $programDetailsUrl = $DETAILS_URL . "/${program_id}"; - print STDERR "Retrieving details for program id $program_id: $programDetailsUrl\n" if ($DEBUG); - my $resp = $browser->get( $programDetailsUrl ); + if ($DEBUG) { + $exec_duration = time() - $start_runtime; + print STDERR "$exec_duration : Retrieving details for program id $program_id: $programDetailsUrl\n"; + } + + my $can_accept = HTTP::Message::decodable; + #my $resp = $browser->get($programDetailsUrl, 'Accept-Encoding' => $can_accept, ); + + my $resp = $browser->get($programDetailsUrl, 'Accept-Encoding' => $can_accept, 'Accept-Language' => "en-US,en;q=0.9",); + + if(! $resp->is_success()) { print STDERR "Error getting program details for $program_id: " . $resp->status_line() . "\n"; + if ($DEBUG) { + $exec_duration = time() - $start_runtime; + print STDERR "$exec_duration : $programDetailsUrl \n"; + } return ""; - } + } - # my $detailContent = $resp->content(); - # my $parser = HTML::TokeParser->new( \$detailContent ); - my $detail_js = decode_json $resp->content(); + my $detail_js = eval { decode_json( $resp->decoded_content() ) }; + if ($@) { + print STDERR "Decoding JSON response failed. \n" if ($VERBOSE); + } my $detail = $detail_js->{"programDetail"}; # Extract program details diff -Nru xmltv-0.6.1/grab/nl/test.conf xmltv-0.6.3/grab/nl/test.conf --- xmltv-0.6.1/grab/nl/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/nl/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,83 +0,0 @@ -# Channel ID Channel name -channel 1 # Nederland 1 -channel 2 # Nederland 2 -# channel 3 # Nederland 3 -# channel 4 # RTL 4 -# channel 5 # Eén -# channel 6 # KETNET/Canvas -# channel 7 # BBC 1 -# channel 8 # BBC 2 -# channel 9 # ARD -# channel 10 # ZDF -# channel 11 # RTL -# channel 12 # WDR Fernsehen -# channel 13 # NDR Fernsehen -# channel 14 # Südwest Fernsehen -# channel 15 # RTBF La 1 -# channel 16 # RTBF La 2 -# channel 17 # TV 5 -# channel 18 # National Geographic -# channel 19 # Eurosport -# channel 20 # TCM -# channel 21 # Cartoon Network -# channel 24 # Film1.1 -# channel 25 # MTV -# channel 26 # CNN -# channel 27 # Rai Uno -# channel 28 # Sat 1 -# channel 29 # Discovery Channel -# channel 31 # RTL 5 -# channel 32 # TRT int. -# channel 33 # Spirit 24 -# channel 34 # Veronica -# channel 35 # TMF -# channel 36 # SBS 6 -# channel 37 # NET 5 -# channel 38 # ARTE -# channel 39 # Film1.2 -# channel 40 # AT 5 -# channel 46 # RTL 7 -# channel 49 # VTM -# channel 50 # 3Sat -# channel 58 # PRO 7 -# channel 59 # 2BE -# channel 60 # VT4 -# channel 64 # Familie 24 -# channel 65 # Animal Planet -# channel 66 # HumorTV 24 -# channel 67 # Consumenten 24 -# channel 69 # Sterren 24 -# channel 70 # Cultura 24 -# channel 73 # Mezzo -# channel 74 # Regionaal -# channel 81 # HollandDoc 24 -# channel 82 # Geschiedenis 24 -# channel 83 # 3voor12 -# channel 84 # Het Gesprek -# channel 86 # BBC World -# channel 87 # TV E -# channel 89 # Nickelodeon -# channel 90 # BVN -# channel 91 # Comedy Central -# channel 92 # RTL 8 -# channel 93 # 13TH STREET -# channel 94 # SCI FI -# channel 99 # Sport1 -# channel 100 # RTV Utrecht -# channel 101 # RTV West -# channel 102 # RTV Rijnmond -# channel 103 # RTV Noord-Holland -# channel 104 # BBC Prime -# channel 105 # Private Spice -# channel 107 # Film1.3 -# channel 108 # RTV Noord -# channel 109 # Omrop Fryslân -# channel 110 # RTV Drenthe -# channel 111 # RTV Oost -# channel 112 # Omroep Gelderland -# channel 113 # Omroep Flevoland -# channel 114 # Omroep Brabant -# channel 115 # L1 TV -# channel 116 # Omroep Zeeland -# channel 148 # Eredivisie Live -# channel 200 # Z@ppelin diff -Nru xmltv-0.6.1/grab/nl/tv_grab_nl xmltv-0.6.3/grab/nl/tv_grab_nl --- xmltv-0.6.1/grab/nl/tv_grab_nl 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/nl/tv_grab_nl 1970-01-01 00:00:00.000000000 +0000 @@ -1,894 +0,0 @@ -#!/usr/bin/perl -w - -eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}' - if 0; # not running under some shell -=pod - -=head1 NAME - -tv_grab_nl - Grab TV listings for Holland. - -=head1 SYNOPSIS - -tv_grab_nl --help - -tv_grab_nl [--config-file FILE] --configure - -tv_grab_nl [--config-file FILE] [--output FILE] [--days N] - [--offset N] [--quiet] [--fast] - -=head1 DESCRIPTION - -Output TV listings for several channels available in Holland. -The data comes from www.tvgids.nl. The grabber relies on -parsing HTML so it might stop working at any time. - -First run B to choose, which channels you want -to download. Then running B with no arguments will output -listings in XML format to standard output. - -B<--configure> Prompt for which channels, -and write the configuration file. - -B<--config-file FILE> Set the name of the configuration file, the -default is B<~/.xmltv/tv_grab_nl.conf>. This is the file written by -B<--configure> and read when grabbing. - -B<--output FILE> Write to FILE rather than standard output. - -B<--days N> Grab N days. The default is one week. - -B<--offset N> Start N days in the future. The default is to start -from today. - -B<--fast> Only fetch summary information for each programme. This is -only title, start/stop times, categories. - -B<--quiet> Suppress the progress messages normally written to standard -error. - -B<--capabilities> Show which capabilities the grabber supports. For more -information, see L - -B<--version> Show the version of the grabber. - -B<--help> Print a help message and exit. - -=head1 SEE ALSO - -L. - -=head1 AUTHOR - -latest patch submitted by Teus Hagen -maintainer was Eric Bus (xmltv@fambus.nl). - -First version by Guido Diepen and Ed Avis (ed@membled.com). -Originally based on tv_grab_fi by Matti Airas. - -=cut - -###################################################################### -# initializations - -use strict; -use XMLTV; -use XMLTV::Version "$XMLTV::VERSION"; -use XMLTV::Capabilities qw/baseline manualconfig cache share/; -use XMLTV::Description 'Holland'; -use Getopt::Long; -use Data::Dumper; -use HTML::TreeBuilder; -use HTML::Entities; # parse entities -use HTTP::Cache::Transparent; -use IO::File; -#use URI; -#use Date::Manip; -use Date::Parse; -use Date::Format; -use DateTime; -use Encode; -use XMLTV::Memoize; -use XMLTV::ProgressBar; -use XMLTV::Ask; -use XMLTV::Config_file; -use XMLTV::DST; -use XMLTV::Get_nice; -use XMLTV::Mode; -use XMLTV::Date; -# Todo: perhaps we should internationalize messages and docs? -use XMLTV::Usage < \$opt_days, - 'offset=i' => \$opt_offset, - 'help' => \$opt_help, - 'configure' => \$opt_configure, - 'config-file=s' => \$opt_config_file, - 'gui:s' => \$opt_gui, - 'output=s' => \$opt_output, - 'quiet' => \$opt_quiet, - 'fast' => \$opt_fast, - 'list-channels' => \$opt_list_channels, - 'share=s' => \$opt_share, # undocumented - 'debug' => \$opt_debug, # undocumented - ) - or usage(0); - -my $opt_slow = !$opt_fast; # configure the old version's 'slow' flag (this is now the default unles --fast is specified) - -if( defined $opt_offset && ( $opt_offset < 0 || $opt_offset > 6 ) ) { - print '!! Offset must be between 0-6 (0 is today), using default (0)',"\n"; - $opt_offset = 0; -} - -if( defined $opt_days && ( $opt_days < 1 || $opt_days + $opt_offset > 7 ) ) { - print '!! Days must be between 1-7, using default (7)',"\n"; - $opt_days = 7 - $opt_offset; -} - -usage(1) if $opt_help; - -# Initialise the web page cache -HTTP::Cache::Transparent::init( { - BasePath => get_default_cachedir(), - NoUpdate => 4*3600, # cache time in seconds - MaxAge => 24, # flush time in hours - Verbose => $opt_debug, -} ); - -XMLTV::Ask::init($opt_gui); - -my $mode = XMLTV::Mode::mode('grab', # default - $opt_configure => 'configure', - $opt_list_channels => 'list-channels', - ); - -# File that stores which channels to download. -my $config_file - = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_nl', $opt_quiet); - -if ($mode eq 'configure') { - XMLTV::Config_file::check_no_overwrite($config_file); - open(CONF, ">$config_file") or die "cannot write to $config_file: $!"; - - # Get channels, and download what we can from the site. When - # configuring it's useful to download at least something as a - # check that the site is reachable etc. - # - my $bar = new XMLTV::ProgressBar('getting list of channels', 1) - if not $opt_quiet; - my %channels = get_channels(); - die 'no channels could be found' if (scalar(keys(%channels)) == 0); - update $bar if not $opt_quiet; - $bar->finish() if not $opt_quiet; - - # Ask about each channel. - my @chs = sort { $a <=> $b } keys %channels; - my @names = map { $channels{$_}->{'name'} } @chs; - my @qs = map { "add channel $_?" } @names; - my @want = ask_many_boolean(1, @qs); - - print CONF "# Channel ID\t\t\t\t Channel name\n"; - - foreach (@chs) { - my $w = shift @want; - my $name = shift @names; - - warn("cannot read input, stopping channel questions"), last - if not defined $w; - # No need to print to user - XMLTV::Ask is verbose enough. - - # Print a config line, but comment it out if channel not wanted. - print CONF '# ' if not $w; - print CONF "channel $_"; - print CONF "\t\t\t\t# $name\n"; - } - - close CONF or warn "cannot close $config_file: $!"; - say("Finished configuration."); - - exit(); -} - -# Not configuring, we will need to write some output. -die if $mode ne 'grab' and $mode ne 'list-channels'; - -# But if grabbing, check the config file is sane before we write -# anything. -# -my @config_lines; -if ($mode eq 'grab') { - @config_lines = XMLTV::Config_file::read_lines($config_file); -} - -my %w_args; -if (defined $opt_output) { - my $fh = new IO::File(">$opt_output"); - die "cannot write to $opt_output: $!" if not defined $fh; - $w_args{OUTPUT} = $fh; -} -$w_args{encoding} = $ENCODING; -my $writer = new XMLTV::Writer(%w_args); -# TODO: standardize these things between grabbers. -$writer->start - ({ 'source-info-url' => 'http://www.tvgids.nl/', - 'source-data-url' => 'http://www.tvgids.nl/', - 'generator-info-name' => 'XMLTV', - 'generator-info-url' => 'http://www.xmltv.org/' - }); - -if ($mode eq 'list-channels') { - my $bar = new XMLTV::ProgressBar('getting list of channels', 1) - if not $opt_quiet; - - # Don't do any page fetches - assume the contents of the channels - # file has already been checked enough. - # - my %channels = get_channels(); - die 'no channels could be found' if (scalar(keys(%channels)) == 0); - update $bar if not $opt_quiet; - - foreach my $ch_did (sort(keys %channels)) { - my $ch_name = $channels{$ch_did}->{'name'}; - my $ch_xid = "$ch_did.tvgids.nl"; - $writer->write_channel({ id => $ch_xid, - 'display-name' => [ [ $ch_name ] ], - 'icon' => [{'src' => $channels{$ch_did}->{'url'}}] }); - } - - $writer->end(); - $bar->finish() if not $opt_quiet; - exit(); -} - -# Not configuring or writing channels, must be grabbing listings. -die if $mode ne 'grab'; -my %channels = get_channels(); -my (@channels, $ch_did, $ch_name); -my $line_num = 0; -my $warned_old_format = 0; -my $bad = 0; - -foreach (@config_lines) { - ++ $line_num; - next if not defined; - - if (/^channel:?\s+(\S+)\s+(.+)/) { - # Old format storing display-names in the config file. Well, - # if they're there we ought to check them. - # - $ch_did = $1; - $ch_name = $2; - $ch_name =~ s/\s*$//; - push @channels, $ch_did; - $channels{$ch_did} = $ch_name; - warn "$config_file format needs upgrading, rerun --configure\n" - unless $warned_old_format++; - } - elsif (/^channel\s+(\S+)\s*$/) { - push @channels, $1; - } - else { - warn "$config_file:$line_num: bad line\n"; - $bad = 1; - } -} -die "$config_file has errors, not continuing\n" if $bad; - -###################################################################### -# begin main program - -# $opt_offset is taken into account later, we don't need to lie about -# $now. This does make it impossible to use --offset together with -# --cache to reuse an old cache file from a few days ago - to do that -# you need to change $now below. But --cache is undocumented so I -# don't consider this a problem. -# -my $now = parse_date('now'); -# Any Date_Init('TZ=UTC') would go here. But it may not be needed -# with parse_local_date(). -# - -my @to_get; - -# We now fetch a complete page per channel -# This page contains all the program links for the specified days -for( my $i = $opt_offset; $i < ($opt_days+$opt_offset); $i++ ) -{ - foreach $ch_did (@channels) { - my $ch_xid = "$ch_did.tvgids.nl"; - #my $url = 'http://www.tvgids.nl/zoeken/'."?q=&d=$i&z=$ch_did&t=0&g=&v=0"; - my $url = "http://www.tvgids.nl/json/lists/programs.php?channels=$ch_did&day=$i"; - - push @to_get, [ $url, $ch_xid, $ch_did, $i ]; - } -} - -# programme store - stores programmes within starttime within channel (so we can check for duplicates) -# @{ $programmes->{ $ch_id }->{ $p->{'start_epoch'} } -my $programmes = {}; - -my $bar = new XMLTV::ProgressBar('Downloading schedules...', scalar @to_get) - if not $opt_quiet; - -# time limits for grab -my $grab_start = DateTime->today(time_zone => 'Europe/Amsterdam')->epoch() + ($opt_offset * 86400); -my $grab_end = $grab_start + ($opt_days * 86400); -#print STDERR "\n start/end grab: $grab_start $grab_end \n"; - -foreach (@to_get) { - my ($url, $ch_xmltv_id, $ch_tvgids_id, $i) = @$_; - die if ref $url; - - #my $start_day = UnixDate(DateCalc($now, "+ $i days"), '%Y-%m-%d'); - - get_programmes ($url, $ch_xmltv_id, $ch_tvgids_id); - - update $bar if not $opt_quiet; -} -$bar->finish() if not $opt_quiet; - - -# All data has been gathered. Restructure the data for XMLTV writer. - # -my @to_write = (); -foreach ( keys %{$programmes} ) { - my $_ch_progs = $programmes->{$_}; - foreach ( sort keys %{$_ch_progs} ) { - my $_dt_progs = $_ch_progs->{$_}; - foreach (@{ $_dt_progs }) { - push @{to_write}, $_; - } - } - } - -# write the channels info -foreach $ch_did (@channels) { - my $ch_name = $channels{$ch_did}->{'name'}; - my $ch_xid = "$ch_did.tvgids.nl"; - $writer->write_channel({ id => $ch_xid, - 'display-name' => [ [ $ch_name ] ], - 'icon' => [{'src' => $channels{$ch_did}->{'url'}}] }); - } - -# write the programmes info -$writer->write_programme($_) foreach @to_write; - - # -$writer->end(); - -###################################################################### -# subroutine definitions - -#my $warned_bad_chars; -#sub tidypage( $ ) { -# for (my $tmp = shift) { -# tr/\221\222/''/; -# if (tr/\012\015\040-\176\240-\377//dc) { -# warn 'removing bad characters' unless $warned_bad_chars++; -# } -# return $_; -# } -#} - -# Remove bad chars from entire page -sub tidypage( $ ) { - my $html = shift; - $html =~ s/(\s) /$1/og; # replace 'space- ' with 'space' - $html =~ s/ / /og; # replace any remaining   with space - $html =~ s/­//og; # delete soft hyphens - return $html; -} - -# Remove bad chars from an element -sub tidy( $ ) { - my $html = shift; - return $html if !defined $html; - $html =~ s/(\s)\xA0/$1/og; # replace 'space- ' with 'space' - $html =~ s/\xA0/ /og; # replace any remaining   with space - $html =~ s/\xAD//og; # delete soft hyphens - return $html; -} - -sub trim( $ ) { - # Remove leading & trailing spaces - $_[0] =~ s/^\s+|\s+$//g; - return $_[0]; -} - -sub dedupe( @ ) { - # Remove duplicates from an array - my %temp_hash = map { $_, 0 } @_; - return keys %temp_hash; -} - -sub dedupepreserve( @ ) { - # Remove duplicates from an array preserving the original sequence - my (@unique, %seen) = ((),()); - foreach ( @_ ) { - next if $seen{ lc( $_ ) }++; - push @unique, $_; - } - return @unique; -} - -sub time_to_str( $ ) { - my $input = shift; - - # Replace months - $input =~ s/\bjanuari\b/1/i; - $input =~ s/\bfebruari\b/2/i; - $input =~ s/\bmaart\b/3/i; - $input =~ s/\bapril\b/4/i; - $input =~ s/\bmei\b/5/i; - $input =~ s/\bjuni\b/6/i; - $input =~ s/\bjuli\b/7/i; - $input =~ s/\baugustus\b/8/i; - $input =~ s/\bseptember\b/9/i; - $input =~ s/\boktober\b/10/i; - $input =~ s/\bnovember\b/11/i; - $input =~ s/\bdecember\b/12/i; - $input =~ s/, / /i; - - if( $input =~ /(\d\d?) (\d{1,2}) (\d{4}) (\d\d:\d\d) *- *(\d\d:\d\d) uur/ ) { - return ( "$3-$2-$1 $4:00", "$3-$2-$1 $5:00" ); - } - elsif( $input =~ /(\d\d?) (\d{1,2}) (\d{4}) - *(\d\d):(\d\d) uur/ ) { - return ( "$4-$3-$2 $4:".($5-30).":00", "$3-$2-$1 $4:$5:00" ); - } - elsif( $input =~ /(\d\d?) (\d{1,2}) (\d{4}) (\d\d):(\d\d) *- uur/ ) { - return ( "$3-$2-$1 $4:$5:00", "$3-$2-$1 $4:".($5+30).":00" ); - } -} - - -# Find the available channels. The new site does have a full list of -# channels, which we can freely fetch. -# -sub get_channels() { - my %channels; - - # Download the full list - my $url = 'http://www.tvgids.nl/json/lists/channels.php'; - - my $chs = get_nice_json($url, '', 1); - foreach (@{$chs}) { - $channels{ $_->{'id'} } = { 'name' => $_->{'name'}, 'url' => 'http://tvgidsassets.nl/iphone/channels/big/'.$_->{'id'}.'b.png' }; - } - - return %channels; -} - - -# Fetch & process a day's schedules for a channel -sub get_programmes( $$$ ) { - - my ($url, $ch_xid, $ch_id) = @_; - die if not defined $url; die if ref $url; - - local $SIG{__WARN__} = sub { - warn "$url: $_[0]"; - }; - - local $SIG{__DIE__} = sub { - die "$url: $_[0]"; - }; - - #print STDERR "Fetching ",$url," ...\n" if not $opt_quiet; - - my $t = get_nice_json($url, '', 1); - if ( not $t->{ $ch_id }) { - warn 'did not find any programmes in page' if not $opt_quiet; - return (); - } - - # fetch each programme from the JSON object - # - # some channels return an array { "2" : [{},{}] } - # whereas other return a hash! { "3" : { "0" : {}, "1" : {} } } - # - my @progs_to_process; - - if ( ref $t->{ $ch_id } eq 'HASH' ) { - @progs_to_process = @{ $t->{ $ch_id } }{ sort { $a <=> $b } keys %{ $t->{ $ch_id } } }; - } elsif ( ref $t->{ $ch_id } eq 'ARRAY' ) { - @progs_to_process = @{ $t->{ $ch_id } }; - } else { - die "unknown type of reply object ".ref $t->{ $ch_id }; - } - - PROG: - foreach my $_p (@progs_to_process) { - # e.g. - #{ - # "db_id": "16291308", - # "titel": "De Wereld Draait Door", - # "genre": "Informatief", - # "soort": "Talkshow", - # "kijkwijzer": "", - # "artikel_id": null, - # "artikel_titel": null, - # "artikel_tekst": null, - # "artikel_foto": null, - # "datum_start": "2014-04-29 19:00:00", - # "datum_end": "2014-04-29 20:00:00" - #}, - #{ - # 'db_id' => '16387388', - # 'titel' => "NOS UEFA Champions League Live, Chelsea - Atl\x{e9}tico Madrid", - # 'genre' => 'Sport', - # 'soort' => 'Voetbal', - # 'kijkwijzer' => '', - # 'artikel_id' => '64576', - # 'artikel_titel' => 'Chelsea jaagt op derde cup op rij', - # 'artikel_tekst' => 'Na een ultraverdedigende 0-0-wedstrijd in Madrid mag Chelsea in eigen huis proberen Atlético Madrid te verslaan. Winst is namelijk genoeg voor een plek in de finale van de Champions League.', - # 'artikel_foto' => 'mourinho_madrid_320.jpg', - # 'datum_start' => '2014-04-30 20:40:00', - # 'datum_end' => '2014-04-30 22:45:00', - # 'is_highlight' => '1' - #}; - - - # Grab data into a hash for this programme - my $p; - $p->{'channel'} = $ch_xid; - $p->{'title'} = $_p->{'titel'}; - $p->{'db_id'} = $_p->{'db_id'}; - - # calc TZ offset for this programme - my $tz = DateTime->from_epoch( epoch=>str2time( $_p->{'datum_start'}, 'GMT' ), time_zone=>'Europe/Amsterdam' )->strftime('%z'); - - $p->{'start_epoch'} = str2time($_p->{'datum_start'}, $tz); - $p->{'stop_epoch'} = str2time($_p->{'datum_end'}, $tz); - $p->{'start'} = time2str( "%Y%m%d%H%M%S %z", $p->{'start_epoch'}, $tz ); - $p->{'stop'} = time2str( "%Y%m%d%H%M%S %z", $p->{'stop_epoch'}, $tz ); - - #print STDERR "$tz : $p->{'start_epoch'} \n"; - - push @{$p->{'genres'}}, trim( $_p->{'genre'} ) if ($_p->{'genre'} ne ''); # (e.g. Film) - put before soort - push @{$p->{'genres'}}, trim( $_p->{'soort'} ) if ($_p->{'soort'} ne ''); - - $p->{'url'} = "http://www.tvgids.nl/programma/".$p->{'db_id'}; - - - # is the start time within range of this grab? - next if ( $p->{'start_epoch'} < $grab_start || $p->{'start_epoch'} >= $grab_end ); - - - # if user wants details then get them from the programme page - if ($opt_slow) { - process_details_page ( \$p ); - } - - # de-dupe the categories - @{ $p->{'genres'} } = dedupepreserve ( @{ $p->{'genres'} } ); - - - # Reformat the data to create the data structure for the programme - my $p_out = {}; - $p_out->{'channel'} = $p->{'channel'}; - $p_out->{'title'} = [[ encode($ENCODING, $p->{'title'}), $LANG ]]; - $p_out->{'start'} = $p->{'start'}; - $p_out->{'stop'} = $p->{'stop'} if (defined $p->{'stop'} && $p->{'stop'} ne ''); - $p_out->{'desc'} = [[ encode($ENCODING, $p->{'desc'}), $LANG ]] if (defined $p->{'desc'} && $p->{'desc'} ne ''); - $p_out->{'sub-title'} = [[ encode($ENCODING, $p->{'sub_title'}), $LANG ]] if (defined $p->{'sub_title'} && $p->{'sub_title'} ne ''); - - $p_out->{'date'} = $p->{'year'} if (defined $p->{'year'} && $p->{'year'} ne ''); - - $p_out->{'rating'} = [[ $p->{'rating'} ]] if (defined $p->{'rating'} && $p->{'rating'} ne ''); - push @{ $p_out->{'rating'}[0] }, ( undef, [{'src' => $p->{'rating_icon'}}] ) if (defined $p->{'rating_icon'} && $p->{'rating_icon'} ne ''); - - $p_out->{'url'} = [ encode($ENCODING, $p->{'url'}) ] if (defined $p->{'url'} && $p->{'url'} ne ''); - $p_out->{'video'}->{'aspect'} = '16:9' if (defined $p->{'widescreen'} && $p->{'widescreen'} == 1); - - # encode the 'credits' - @{ $p->{'directors'} } = map { encode($ENCODING, trim $_) } @{ $p->{'directors'} } if defined $p->{'directors'}; - @{ $p->{'actors'} } = map { encode($ENCODING, trim $_) } @{ $p->{'actors'} } if defined $p->{'actors'}; - $p_out->{'credits'}{'director'} = $p->{'directors'} if (defined $p->{'directors'} && scalar $p->{'directors'}); - $p_out->{'credits'}{'actor'} = $p->{'actors'} if (defined $p->{'actors'} && scalar $p->{'actors'}); - - - push @{ $p_out->{'title'} }, [ encode($ENCODING, $p->{'original_title'}) ] if (defined $p->{'original_title'} && $p->{'original_title'} ne ''); # original title is not Dutch! - - # any extra description? - push @{ $p_out->{'desc'} }, [ encode($ENCODING, $p->{'desc_add'}), $LANG ] if (defined $p->{'desc_add'} && $p->{'desc_add'} ne ''); - - - if ( scalar @{ $p->{'genres'} } ) { - foreach (@{ $p->{'genres'} }) { - push @{ $p_out->{'category'} }, [ encode($ENCODING, $_), $LANG ] if ($_ ne ''); - } - } - - - # store the programme avoiding duplicates - # also check for duplicate start times and set clumpidx - { - if ( defined $programmes->{ $ch_id }->{ $p->{'start_epoch'} } ) { - # duplicate prog or contemporary? - my $dup = 0; my $_P; - foreach $_P ( @{ $programmes->{ $ch_id }->{ $p->{'start_epoch'} } } ) { - $dup = 1 if ( $_P->{'title'}[0][0] eq $p_out->{'title'}[0][0] ); # duplicate - } - next PROG if $dup; # ignore duplicates (go to next programme) - if (!$dup) { - # contemporary programme so set clumpidx - my $numclumps = scalar @{ $programmes->{ $ch_id }->{ $p->{'start_epoch'} } } + 1; - # set (or adjust) clumpidx of existing programmes - my $i = 0; - foreach $_P ( @{ $programmes->{ $ch_id }->{ $p->{'start_epoch'} } } ) { - $_P->{'clumpidx'} = "$i/$numclumps"; - $i++; - } - # set clumpidx for new programme - $p_out->{'clumpidx'} = "$i/$numclumps"; - } - } - } - - # store the programme - push @{ $programmes->{ $ch_id }->{ $p->{'start_epoch'} } }, $p_out; - } - - } - -# Fetch and parse an individual programme information page -sub process_details_page( $ ) { - my ($p) = @_; - - my $url = $$p->{'url'}; - - local $SIG{__WARN__} = sub { - warn "$url: $_[0]" unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/); - }; - - # Get HTML::TreeBuilder object. - my $t; - # eval { $t = get_nice_tree $url, \&tidypage }; - # - # page is "charset=iso-8859-1" but sometimes contains UTF-8 ! - # - # website has changed 2015-06-23 - now requires a session cookie for opt-in to storing cookies. Adding the GET param fixes most of the - # detail pages but doesn't fix the 302 redirects. Creating a cookiejar (with or without a preset 'cookieoptin' cookie) doesn't fix this which is odd. - # So we still end up with a few WARN messages for 'did not see 'prog-content' element' when the details page is 302'd to a different URI. - $t = get_nice_tree($url . '?cookieoptin=true'); - if ($@) { - warn "error getting/parsing $url: $@" if not $opt_quiet; - return; - } - - # description is in

    elements in

    (note: there may be >1) - my $_div = $t->look_down('_tag' => 'div', 'id' => 'prog-content'); - if (not $_div) { - warn "did not see 'prog-content' element, skipping page" if not $opt_quiet; - return; - } - { - if ( my $_p = $_div->look_down('_tag' => 'p') ) { - if ( my $_s = $_p->look_down('_tag' => 'span', 'class' => qr/articleblock/ ) ) { - # are there ever >1 genres here? - push @{$$p->{'genres'}}, ( ucfirst(lc( trim( $_s->as_text() ) )) ); - $_s->detach(); - } - $$p->{'desc'} = trim( tidy( $_p->as_text() ) ); - $_p->detach(); - } - # see if there's any more - if ( my $_p = $_div->look_down('_tag' => 'p') ) { - my $html = $_p->as_HTML(); - $html =~ s/(
    |
    )/ /sg; # insert spaces between paragraphs - $html =~ s/<.+?>//sg; # dump the html - $$p->{'desc_add'} = trim( tidy( $html ) ); - } - } - - # get the programme details - $_div = $t->look_down('_tag' => 'div', 'class' => 'programmering_details'); - if (not $_div) { - warn "did not see 'programmering_details' element, skipping" if not $opt_quiet; - return; - } - { - if ( my $_ul = $_div->look_down('_tag' => 'ul') ) { - my @_li = $_ul->look_down('_tag' => 'li'); - # - # DEPRECATED - # e.g. - #
  • Titel:Elysium
    • - #
  • Zender:HBO 1
    • - #
  • Datum:29 april 2014
    • - #
  • Uitzendtijd:18:40 - 20:30 uur
    • - #
  • Genre:Film
    • - # - foreach my $_li (@_li) { - my $li = tidy( $_li->as_text() ); - - if ( $li =~ /Genre:(.*)/ ) { - push @{$$p->{'genres'}}, $1; - } - - if ( $li =~ /Orginele titel:(.*)/ ) { - $$p->{'original_title'} = $1; - } - } - } - } - - $_div = $t->look_down('_tag' => 'div', 'class' => 'programmering_info_detail'); - if (not $_div) { - warn "did not see 'programmering_info_detail' element, skipping" if not $opt_quiet; - return; - } - { - if ( my $_ul = $_div->look_down('_tag' => 'ul') ) { - my @_li = $_ul->look_down('_tag' => 'li'); - # - # DEPRECATED - # e.g. - #
  • Jaar van premiere:2013
    • - #
  • Regisseur:Neill Blomkamp
    • - #
  • Scenario schrijver:Neill Blomkamp
    • - #
  • Componist:Ryan Amon, Kristian Bailey, Craig Berkey, Hennie Britton, Ricardo Cabrera, Michelle Child
    • - #
  • Bijzonderheden:Breedbeeld uitzending
    • - #
  • Acteurs:Matt Damon, Jodie Foster, Sharlto Copley, Alice Braga, Diego Luna
    • - #
  • Niet voor personen tot 16 jaar
    • - # - foreach my $_li (@_li) { - my $li = tidy( $_li->as_text() ); - - if ( $li =~ /Jaar.*:(.*)/ ) { - $$p->{'year'} = $1; - next; - } - - if ( $li =~ /Regisseur.*:(.*)/ ) { - @{ $$p->{'directors'} } = split(/,/, $1); - next; - } - - if ( $li =~ /Acteurs.*:(.*)/ ) { - @{ $$p->{'actors'} } = split(/,/, $1); - next; - } - - if ( $li =~ /Bijzonderheden:.*Breedbeeld.*/i ) { - $$p->{'widescreen'} = 1; - } - - if ( my $_img = $_li->look_down('_tag' => 'img') ) { - my $src = $_img->attr('src'); - if ( $src =~ /kijkwijzer\/(AL|\d{0,2})_/i ) { # e.g. 16_transp.png, Kijkwijzer system has AL,6,9,12,16 - $$p->{'rating'} = uc($1); - $$p->{'rating_icon'} = $src; - } - } - } - } - } - -} - -# Parse date strings that are in Dutch. 'Why not just call -# Date_Init("Language=Dutch")?' I hear you ask. The trouble is that -# Date::Manip's language is a global setting and having set it to -# Dutch we cannot use code that expects English - either in this file -# or in any libraries. The least insane way to proceed is to turn -# Dutch to English strings here. -# -# The conversions to make, however, are swiped from the Date::Manip -# code. -# -sub parse_dutch_date( $ ) { - for (my $tmp = $_[0]) { - s/\bjanuari\b/January/i; - s/\bjan\b/January/i; - s/\bfebruari\b/February/i; - s/\bfeb\b/February/i; - s/\bmaart\b/March/i; - s/\bmaa\b/March/i; - s/\bmrt\b/March/i; - s/\bapril\b/April/i; - s/\bapr\b/April/i; - s/\bmei\b/May/i; - s/\bmei\b/May/i; - s/\bjuni\b/June/i; - s/\bjun\b/June/i; - s/\bjuli\b/July/i; - s/\bjul\b/July/i; - s/\baugustus\b/August/i; - s/\baug\b/August/i; - s/\bseptember\b/September/i; - s/\bsep\b/September/i; - s/\boctober\b/October/i; - s/\boktober\b/October/i; - s/\boct\b/October/i; - s/\bokt\b/October/i; - s/\bnovember\b/November/i; - s/\bnov\b/November/i; - s/\bdecember\b/December/i; - s/\bdec\b/December/i; - - s/\bZondag\b/Sunday/gi; - s/\bMaandag\b/Monday/gi; - s/\bDinsdag\b/Tuesday/gi; - s/\bWoensdag\b/Wednesday/gi; - s/\bDonderdag\b/Thursday/gi; - s/\bVrijdag\b/Friday/gi; - s/\bZaterdag\b/Saturday/gi; - - my $r; - eval { $r = parse_local_date($_, $TZ) }; - die "could not parse date $_ (from Dutch $_[0])" - if $@; - return $r; - } -} - -# Get the user's home directory -sub get_default_dir { - my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH} - if defined( $ENV{HOMEDRIVE} ) - and defined( $ENV{HOMEPATH} ); - - my $home = $ENV{HOME} || $winhome || "."; - return $home; -} - -# Set default cache dir = $HOME/.xmltv/cache -sub get_default_cachedir { - return get_default_dir() . "/.xmltv/cache"; -} diff -Nru xmltv-0.6.1/grab/pt_vodafone/test.conf xmltv-0.6.3/grab/pt_vodafone/test.conf --- xmltv-0.6.1/grab/pt_vodafone/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/pt_vodafone/test.conf 2020-09-07 15:02:53.000000000 +0000 @@ -1,195 +1,217 @@ -channel=382 -channel=20 -channel=345 -channel=352 -channel=375 -channel!408 -channel!3 -channel!2 -channel!36 -channel!30 -channel!386 -channel!13 -channel!12 -channel!27 -channel!14 -channel!356 -channel!236 -channel!158 -channel!173 -channel!48 -channel!387 -channel!126 -channel!195 -channel!73 -channel!488 -channel!342 -channel!198 -channel!471 -channel!94 -channel!241 -channel!157 -channel!105 -channel!47 -channel!228 -channel!480 -channel!59 -channel!76 -channel!45 -channel!50 -channel!487 -channel!85 -channel!412 -channel!153 -channel!22 -channel!11 -channel!31 -channel!475 -channel!390 -channel!95 -channel!341 -channel!92 -channel!149 -channel!160 -channel!88 -channel!200 -channel!485 -channel!93 -channel!51 -channel!364 -channel!17 -channel!52 -channel!481 -channel!199 -channel!8 -channel!363 -channel!154 -channel!368 -channel!6 -channel!7 -channel!111 -channel!243 -channel!15 -channel!151 -channel!4 -channel!227 -channel!360 -channel!172 -channel!10 -channel!410 -channel!112 -channel!161 -channel!100 -channel!372 -channel!40 -channel!69 -channel!68 -channel!486 -channel!54 -channel!489 -channel!403 -channel!43 -channel!108 -channel!414 -channel!128 -channel!230 -channel!42 -channel!21 -channel!188 -channel!18 -channel!479 -channel!38 -channel!113 -channel!110 -channel!155 -channel!239 -channel!407 -channel!9 -channel!405 -channel!34 -channel!238 -channel!5 -channel!61 -channel!41 -channel!174 -channel!171 -channel!150 -channel!33 -channel!349 -channel!196 -channel!476 -channel!130 -channel!129 -channel!183 -channel!107 -channel!66 -channel!46 -channel!344 -channel!193 -channel!413 -channel!409 -channel!231 -channel!374 -channel!125 -channel!53 -channel!350 -channel!91 -channel!385 -channel!490 -channel!391 -channel!373 -channel!415 -channel!56 -channel!206 -channel!145 -channel!16 -channel!348 -channel!347 -channel!19 -channel!39 -channel!28 -channel!82 -channel!81 -channel!181 -channel!127 -channel!240 -channel!63 -channel!62 -channel!89 -channel!159 -channel!417 -channel!104 -channel!144 -channel!83 -channel!35 -channel!351 -channel!355 -channel!86 -channel!184 -channel!37 -channel!182 -channel!101 -channel!26 -channel!202 -channel!84 -channel!103 -channel!106 -channel!25 -channel!177 -channel!24 -channel!23 -channel!152 -channel!416 -channel!245 -channel!392 -channel!371 -channel!55 -channel!478 -channel!146 -channel!380 -channel!58 -channel!197 -channel!194 -channel!1 -channel!244 -channel!80 +channel=RTP 1 HD +channel=RTP 2 +channel=SIC +channel=TVI HD +channel!SIC N +channel!RTP 3 +channel!TVI24 HD +channel!CMTV +channel!SPTV+ +channel!GLOBO +channel!CANAL 11 HD +channel!PORTOC HD +channel!SIC CARAS +channel!SIC R +channel!SIC M +channel!RTPMEM +channel!24K HD +channel!Canal Q HD +channel!SPTV 1 +channel!SPTV 2 +channel!SPTV 3 +channel!SPTV 4 +channel!SPTV 5 +channel!SPTV+ HD +channel!SPTV1 HD +channel!SPTV2 HD +channel!SPTV3 HD +channel!SPTV4 HD +channel!SPTV5 HD +channel!ABOLA TV HD +channel!BTV1 +channel!BTV1 HD +channel!SCP TV +channel!SCP TV HD +channel!SPTV NBA +channel!EURSP +channel!EURSPHD +channel!EURSP2 +channel!EURSP2 HD +channel!ES1 HD +channel!ES2 HD +channel!ES3 HD +channel!ES4 HD +channel!ES5 HD +channel!ES6 HD +channel!FuelTVHD +channel!PFC +channel!DISNEY +channel!DISNEY J +channel!CN PT +channel!PANDA +channel!PANDA B +channel!BABYTV +channel!SIC K +channel!BOOMERANG +channel!SUPER RTL +channel!TVC 1 +channel!TVC 2 +channel!TVC 3 +channel!TVC 4 +channel!TVC1 HD +channel!TVC2 HD +channel!TVC 3 HD +channel!TVC 4 HD +channel!TVC S HD +channel!TVC S +channel!HOLLYW HD +channel!HOLLYW +channel!CINEMUNDO HD +channel!CINEMUNDO SD +channel!Fox M HD +channel!FOX M +channel!FOXHD +channel!FOX +channel!AXN HD +channel!AXN +channel!FOXLHD +channel!FOXLIF +channel!Fox C HD +channel!FOXCRI +channel!FOX Comedy SD +channel!FOX Comedy HD +channel!AXN W HD +channel!AXN W +channel!AXN B HD +channel!AXN BLK +channel!SyFy HD +channel!SyFy SD +channel!AMC HD +channel!AMC SD +channel!MTV P HD +channel!MTV PT +channel!VH1 +channel!CLASSICA +channel!DJAZZ +channel!MCM TOP SD +channel!MCMTOP +channel!MCM POP +channel!CMUSIC +channel!MEZZO +channel!AFRO M +channel!TRACE HD +channel!TRACE +channel!TRACE TOCA +channel!MEZZO LIVE HD +channel!S+ +channel!Disc HD +channel!DISCOV +channel!NGC HD +channel!NGC +channel!NGW HD +channel!NGCWIL +channel!HISTOR +channel!ODISSE +channel!ODISSEIA HD +channel!DOCUBOX HD +channel!BLAZE +channel!BLAZE HD +channel!TLC +channel!E! HD +channel!E! +channel!TRAVEL HD +channel!TRAVEL +channel!FASHION TV HD +channel!FASHTV +channel!TFN HD +channel!TFN +channel!24 K +channel!BBC ENTERT +channel!MZENHD +channel!LUXEHD +channel!Canal Q +channel!GINX +channel!QYOU HD +channel!QYOU SD +channel!GLOBO NOW HD +channel!GLOBO NOW +channel!TV REC HD +channel!TV REC +channel!EURONEWS EN +channel!CNN +channel!EURONW +channel!BLOOMB +channel!Sky News +channel!BBC WD +channel!CNBC E +channel!ALJAZE +channel!RAINEWS +channel!PHOENI +channel!TVE 24 +channel!DW +channel!FRA24E +channel!FRA24F +channel!REC NEWS +channel!CMTV HD +channel!180 +channel!180 HD +channel!ABOLA TV +channel!CANAL 11 +channel!RTPMAD +channel!RTPACOR +channel!PORTOC +channel!CNova +channel!AR TV +channel!Local HD +channel!Localv +channel!RTP AF +channel!TPA +channel!TVC NEWS +channel!TCV INT +channel!UATV +channel!VOX +channel!RTL +channel!TVGalicia +channel!RTP 1 +channel!TVEi +channel!M6 +channel!TV5MON +channel!KBS WORLD +channel!RAI 1 +channel!PROTV +channel!RUSSIA +channel!RT DOC +channel!CUBAVI +channel!NHK WORLD +channel!KURIAKOS TV +channel!ARIRANG TV +channel!EUROCHANNEL PT +channel!EUROCHANNEL FR +channel!TOUROS +channel!CAC & PESCA +channel!CACVISION HD +channel!CACVISION +channel!SET A +channel!STAR P +channel!SET M +channel!STAR G +channel!NAUTICAL +channel!FightBox +channel!FastnFunBox +channel!FIGHT NETWORK HD +channel!FUELTV +channel!DOGTV +channel!SIC HD +channel!SIC N HD +channel!GLOBO HD +channel!SIC M HD +channel!SIC C HD +channel!SIC R HD +channel!SIC K HD +channel!SPTV 4K +channel!RTP1 H +channel!RTP 2 HD +channel!SIC H +channel!TVI +channel!TVI24 +channel!RTP HD diff -Nru xmltv-0.6.1/grab/pt_vodafone/tv_grab_pt_vodafone xmltv-0.6.3/grab/pt_vodafone/tv_grab_pt_vodafone --- xmltv-0.6.1/grab/pt_vodafone/tv_grab_pt_vodafone 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/pt_vodafone/tv_grab_pt_vodafone 2020-09-07 15:02:53.000000000 +0000 @@ -1,10 +1,12 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl =pod +=encoding utf8 + =head1 NAME -tv_grab_pt_vodafone - Grab TV listings for Vodafone in Portugal. +tv_grab_pt_vodafone - Grab TV listings for Vodafone in Portugal =head1 SYNOPSIS @@ -13,24 +15,24 @@ tv_grab_pt_vodafone --configure [--config-file FILE] tv_grab_pt_vodafone [--config-file FILE] - [--days N] [--offset N] [--channel xmltvid,xmltvid,...] - [--output FILE] [--quiet] [--debug] + [--days N] [--offset N] [--channel xmltvid,xmltvid,...] + [--output FILE] [--quiet | --debug] tv_grab_pt_vodafone --list-channels [--config-file FILE] - [--output FILE] [--quiet] [--debug] + [--output FILE] [--quiet | --debug] =head1 DESCRIPTION -Output TV and listings in XMLTV format for many stations -available in Portugal. This program consumes the EPG service -from Vodafone at L. +Output TV listings in XMLTV format for many stations available in Portugal. +This program consumes the EPG service from Vodafone at +L. First you must run B to choose which stations you want to receive. -Then running B with no arguments will get a listings for -the stations you chose for all available days including today. +Then running B with no arguments will get listings for +the stations you chose for the maximum 7 days, including today. =head1 OPTIONS @@ -50,11 +52,11 @@ B<--quiet> Only print error-messages on STDERR. -B<--debug> Provide more information on progress to stderr to help in +B<--debug> Provide more information on progress to STDERR to help in debugging. -B<--list-channels> Output a list of all channels that data is available - for. The list is in xmltv-format. +B<--list-channels> Output a list of all channels that data is available for. +The list is in xmltv-format. B<--capabilities> Show which capabilities the grabber supports. For more information, see L @@ -65,8 +67,8 @@ =head1 ERROR HANDLING -If the grabber fails to download data from webstep, it will print an -errormessage to STDERR and then exit with a status code of 1 to indicate +If the grabber fails to download data from Vodafone, it will print an +error message to STDERR and then exit with a status code of 1 to indicate that the data is missing. =head1 ENVIRONMENT VARIABLES @@ -76,17 +78,21 @@ it might be necessary to set HOME to a path without spaces in it. =head1 CREDITS + Kevin Groeneveld (kgroeneveld at gmail dot com) -This grabber uses code from tv_grab_pt_meo by Karl Dietz, dekarl -at- users -dot- sourceforge -dot- net. -This grabber uses code from tv_grab_zz_sdjson by Kevin Groeneveld, kgroeneveld -at- gmail -dot- com. -The original idea of this grabber came from higuita's shell script, -see L. -Special thanks to Vodafone, for building a clean, fast and public access API, -much more reliable than Meo open API ( but sadly not as open) and much better -than lack of any API from NOS. +This grabber uses code from tv_grab_pt_meo by Karl Dietz, dekarl -at- users -dot- sourceforge -dot- net, +and from tv_grab_zz_sdjson by Kevin Groeneveld, kgroeneveld -at- gmail -dot- com. + +The original idea of this grabber came from higuita's shell script, see +L. + +Special thanks to Vodafone for building a clean, fast, and public access API; +much more reliable than Meo's open API (but sadly not as open) and much better +than the lack of any API from NOS. =head1 AUTHOR + Nuno Sénica, nsenica -at- gmail -dot- com. =head1 BUGS @@ -95,11 +101,13 @@ =cut +use warnings; use strict; use utf8; use XMLTV; use XMLTV::Version "$XMLTV::VERSION"; use DateTime; +use DateTime::Format::Strptime; use Encode; # used to convert 'perl strings' into 'utf-8 strings' use XML::LibXML; use XMLTV::Configure::Writer; @@ -107,18 +115,22 @@ use XMLTV::Options qw/ParseOptions/; use JSON; use URI::Escape qw/ uri_escape /; +use URI::Encode qw/ uri_encode uri_decode/; #use Data::Dump qw/pp/; # uncomment to debug -my $maxdays = 1+7; # data source is limited to n days (including today) +my $maxdays = 1+6; # data source is limited to 7 days (including today) my $grabber_name = 'tv_grab_pt_vodafone'; -my $grabber_version = '1.00'; +my $grabber_version = '2.00'; -my $json_baseurl = 'https://tvnetvoz.vodafone.pt'; -my $json_api = '/sempre-consigo/'; -$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0; +my $json_baseurl = 'https://web.ott-red.vodafone.pt'; +my $json_api = '/ott3_webapp/'; -my $ua = LWP::UserAgent->new(agent => "$grabber_name $grabber_version"); +my $ua = LWP::UserAgent->new(ssl_opts => { + verify_hostname => 0, + SSL_version => 'TLSv12:!SSLv2:!SSLv3:!TLSv1:!TLSv11', +}); +$ua->agent("$grabber_name $grabber_version"); $ua->default_header('accept-encoding' => scalar HTTP::Message::decodable()); my( $opt, $conf ) = ParseOptions( { @@ -208,9 +220,9 @@ # Return a string containing an xmltv-document with -elements # for all available channels. - my $channellist=json_request( 'get', 'datajson/epg/channels.jsp' ); + my $channellist=json_request( 'get', 'v1/channels' ); - $channellist = $channellist->{result}->{channels}; + $channellist = $channellist->{data}; my $output=XML::LibXML::Document->new( '1.0', 'utf-8' ); my $root=XML::LibXML::Element->new( 'tv' ); @@ -234,51 +246,72 @@ { my( $writer, $startDate, $endDate ) = @_; - my $baseRequest = 'epg.do?action=getPrograms'; + my $baseRequest = 'v1.5/programs/grids/'; my @channelList = @{$conf->{channel}}; my $curDate = $startDate; my %xmlchannels; my %xmlprogs; - while ($curDate < $endDate) { - print( STDERR "requesting EPG for ".$curDate->ymd()."\n" ) if( !$opt->{quiet} ); - print( STDERR " POST ".$json_baseurl.$json_api.$baseRequest." payload: chanids=".(join ",",@channelList)."&day=".$curDate->ymd()."\n" ) if( $opt->{debug} ); + my $nr_days = $startDate->delta_days($endDate)->days; - my $epgSource = json_request('post', $baseRequest, 'chanids='.(join ",",@channelList).'&day='.$curDate->ymd()); + for my $day (0..$nr_days) { - if ( $epgSource->{result} == 500 ){ - die("Bad EPG download, probably channel list is outdated, rerun the grabber configure to update the list.\n" ); - }; + my $epg_day = $day + $opt->{offset}; - for my $channel ( @{ $epgSource->{result}->{channels} || [] }) { - #pp( $channel ) if( $opt->{debug} ); # uncomment to debug + for my $channel (@channelList) { - my $channelId = make_channelid( $channel->{callLetter}); + my $encoded_channel = uri_encode($channel); + + print( STDERR "requesting EPG from " . $curDate->ymd() . " for " . $encoded_channel . "\n" ) if( !$opt->{quiet} ); + print( STDERR " GET ".$json_baseurl.$json_api.$baseRequest.$encoded_channel."/".$epg_day."\n" ) if( $opt->{debug} ); + + my $epgSource = json_request('get', $baseRequest."/".$encoded_channel."/".$epg_day); + + if ( ! $epgSource ){ + die("Bad EPG download, probably channel list is outdated, rerun the grabber configure to update the list.\n" ); } + elsif ( $epgSource->{data}->@* == 0 ){ + print( STDERR " Empty EPG download for ".$channel.", probably channel list is outdated or no API data for that channel\n" . + " Rerun the grabber configure to update the list or check for the channel EPG in the Vodafone app.\n" ); + next; + }; + + my $data = $epgSource->{data}; + + my $channelId = make_channelid( $data->[0]->{channel}->{id} ); my %ch = ( - 'display-name' => [ [ sanitizeUTF8($channel->{name}), 'pt' ] ], 'id' => $channelId, - 'icon' => [ { src => "https://tvnetvoz.vodafone.pt/sempre-consigo/imgs?action=logo_channel_tablet_details&chanid=" .$channel->{id}. "&mime=true&no_default=false" } ], + 'icon' => [ { src => $data->[0]->{channel}->{logo} } ], ); + # multiple display-names are ok and may be useful to match other tools lists + my @displayname = ( [ sanitizeUTF8( $data->[0]->{channel}->{name} ), 'pt' ] , + [ sanitizeUTF8( $data->[0]->{channel}->{id} ), 'pt' ] ); - $xmlchannels{ $channelId } = \%ch ; + push @{ $ch{'display-name'} }, @displayname ; - for my $programme ( @{ $channel->{programList} }) { + $xmlchannels{ $channelId } = \%ch ; + PROGRAMME: + for my $programme ( @{ $data }) { my %prog; - $prog{channel} = $channelId; - $prog{title} = [ [ parse_title(\%prog, sanitizeUTF8($programme->{programTitle})), 'pt' ] ]; - $prog{desc} = [ [ sanitizeUTF8($programme->{programDetails}), 'pt' ] ]; - if ($programme->{pid}) { - $prog{icon} = [ { src => "http://web.ottimg.vodafone.pt/iptvimageserver/Get/" .uri_escape($channel->{callLetter}). "_".$programme->{pid}."/16_9/325/244" } ]; - } - - my ($dtstart, $dtend) = make_dates($programme->{date}, $programme->{startTime}, $programme->{endTime}, $programme->{duration}); + my ($dtstart, $dtend, $starts_today) = make_dates($programme->{startTime}, $programme->{endTime}, $curDate); + next PROGRAMME unless $starts_today; $prog{start} = $dtstart; $prog{stop} = $dtend; + $prog{channel} = $channelId; + $prog{title} = [ [ sanitizeUTF8($programme->{title}), 'pt' ] ]; + $prog{desc} = [ [ sanitizeUTF8($programme->{description}), 'pt' ] ] if ($programme->{description}); + $prog{length} = ( $programme->{duration} ) if ($programme->{duration}); + $prog{icon} = [ { src => $programme->{image} } ] if ($programme->{image}); + + $prog{category} = [ [ sanitizeUTF8($programme->{channel}{category}), 'pt' ] ] if ($programme->{channel}{category}); + $prog{'sub-title'} = [ [ sanitizeUTF8($programme->{series}{episodeTitle}), 'pt' ] ] if ($programme->{series}->{episodeTitle}) ; + + $prog{'episode-num'} = make_episode_num($programme); + # We can get the same programme for two different days if it goes past midnight. # Lets remove duplicates here. $xmlprogs{$channelId}{ $dtstart, $dtend } = \%prog; @@ -286,6 +319,7 @@ } } + $curDate->add( days => 1); } @@ -300,8 +334,6 @@ $str =~ s/[^[:print:]]+//g; -# my $octets = decode('UTF-8', $str, Encode::FB_DEFAULT); - return encode('UTF-8', $str, Encode::FB_CROAK); } @@ -321,7 +353,7 @@ push(@params, content => $content) if defined $content; my $response = $ua->$method($url, @params); if($response->is_success()) { - return JSON->new->utf8(0)->decode( $response->decoded_content()); + return JSON->new->utf8(1)->decode( $response->decoded_content()); } else { my $msg = $response->decoded_content(); @@ -333,33 +365,69 @@ ."$error->{'message'} ($error->{'code'}/$error->{'response'})"; } - die $msg, "\n"; + print( STDERR " Error on the remote EPG API call\n" ) if( !$opt->{quiet} ); + print( STDERR $msg . "\n" ) if( $opt->{debug} ); + + return JSON->new->utf8(1)->decode('{"data": [] }'); } } +sub make_episode_num +{ + my ($programme) = @_; + + return unless $programme->{series}; + + my $output; + + my $season; + my $episode; + + if ( $programme->{series}{season} ) { + $season = $programme->{series}{season} - 1; + } + + if ( $programme->{series}{episode} ) { + $episode = $programme->{series}{episode} - 1; + } + + $output = [ [ ($season // "") . "." . ($episode // "") . ".", 'xmltv_ns' ] ] if (defined $season || defined $episode); + + my $seasonLabel = sanitizeUTF8($programme->{series}{seasonLabel}) if ($programme->{series}{seasonLabel}) ; + my $episodeLabel = sanitizeUTF8($programme->{series}{episodeLabel}) if ($programme->{series}{episodeLabel}) ; + + if ( defined $seasonLabel && defined $episodeLabel ) { + push @{ $output }, [ $seasonLabel ." ". $episodeLabel , 'onscreen' ]; + } + elsif ( defined $seasonLabel ) { + push @{ $output }, [ $seasonLabel , 'onscreen' ]; + } + elsif ( defined $episodeLabel ) { + push @{ $output }, [ $episodeLabel , 'onscreen' ]; + } + return $output; + +} sub make_dates { - my( $date, $startTime, $endTime, $duration ) = @_; + my( $startTime, $endTime, $curDate ) = @_; - my($day, $month, $year) = - ($date =~ m/(\d{2})-(\d{2})-(\d{4})/); + my $strp = new DateTime::Format::Strptime( pattern => '%FT%TZ' ); - my ($startHour, $startMinute) = ( $startTime =~ m/(\d{1,2}):(\d{2})/); - my ($endHour, $endMinute) = ( $endTime =~ m/(\d{1,2}):(\d{2})/); + my $dtstart = $strp->parse_datetime($startTime); + my $starts_today = 0; + # does the programme start on the day we want listings for? + if ($dtstart->day == $curDate->day) { + $starts_today = 1; + } - my $dtstart = DateTime->new( year => $year, - month => $month, - day => $day, - hour => $startHour, - minute => $startMinute, - second => 0, - time_zone => 'Europe/Lisbon', - ); + my $dtend = $strp->parse_datetime($endTime); - my $dtend = $dtstart->clone()->add(minutes => $duration); + # dates look like GMT, we tried UTC but in summer time they fail + #return ($dtstart->strftime( '%Y%m%d%H%M%S %z' ), $dtend->strftime( '%Y%m%d%H%M%S %z' )); + return ($dtstart->strftime( '%Y%m%d%H%M%S +0000' ), $dtend->strftime( '%Y%m%d%H%M%S +0000' ), $starts_today); - return ($dtstart->strftime( '%Y%m%d%H%M%S %z' ), $dtend->strftime( '%Y%m%d%H%M%S %z' )); } sub make_channelid @@ -374,29 +442,3 @@ $id .= '.tv.vodafone.pt'; # append domain part return( $id ); } - -sub parse_title -{ - my $prog = shift; - my $title = shift; - - if (!defined ($title)) { - return undef; - } - - if ($title =~ m/(\s+|:)T\d+\s+-?\s*Ep\.\s*\d+$/) { - # found season and episode in title - my ($delimiter,$season, $episode) = ($title =~ m/(\s+|:)T(\d+)\s+-?\s*Ep\.\s*(\d+)$/); - # uncomment to get the simplified title, without season and episode - #$title =~ s/(\s+|:)T\d+\s+-?\s*Ep\.\s*\d+$//; - $prog->{'episode-num'} = [ [ ($season - 1).'.'.($episode-1).'.', 'xmltv_ns' ] ]; - } elsif ($title =~ m/\s+Ep\.\s*\d+$/) { - # found episode in title - my ($episode) = ($title =~ m/\s+Ep\.\s*(\d+)$/); - # uncomment to get the simplified title, without season and episode - #$title =~ s/\s+Ep\.\s*\d+$//; - $prog->{'episode-num'} = [ [ '.'.($episode-1).'.', 'xmltv_ns' ] ]; - } - - return $title; -} diff -Nru xmltv-0.6.1/grab/se_tvzon/test.conf xmltv-0.6.3/grab/se_tvzon/test.conf --- xmltv-0.6.1/grab/se_tvzon/test.conf 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/se_tvzon/test.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,961 +0,0 @@ -root-url=http://xmltv.xmltv.se/channels.xml.gz -cachedir=/tmp/.xmltv/cache -channel!1.bluemovie.de -channel!13thstreet.de -channel!1bar.dazn.de -channel!2.bluemovie.de -channel!2.eurosport.de -channel!20.mediaset.it -channel!2bar.dazn.de -channel!2extra.eurosport.de -channel!3.bluemovie.de -channel!360tunebox.spi.pl -channel!3plus.ee -channel!3plus.lv -channel!3plus.tv -channel!3sat.de -channel!4plus.tv -channel!5plus.tv -channel!6-eren.dk -channel!abc.tvp.pl -channel!action.cmore.dk -channel!action.cmore.fi -channel!action.cmore.no -channel!action.cmore.se -channel!action.film.viasat.dk -channel!action.film.viasat.fi -channel!action.film.viasat.no -channel!action.film.viasat.se -channel!action.sky.de -channel!actionhd.film.viasat.se -channel!actionhd.sky.de -channel!adultchannel.co.uk -channel!ae-tv.de -channel!alfatv.fi -channel!animalplanet.discovery.de -channel!animalplanet.discovery.dk -channel!animalplanet.discovery.eu -channel!animalplanet.discovery.fi -channel!animalplanet.discovery.no -channel!animalplanet.se -channel!ar.france24.com -channel!arte.de -channel!arthouse.spi.pl -channel!arts.sky.de -channel!asia.tve.es -channel!asien.dw.de -channel!at.viva.tv -channel!atlantic.sky.de -channel!atlantichd.sky.de -channel!atlanticp1.sky.de -channel!atv.at -channel!atv2.at -channel!auto-motor-und-sport.tv -channel!ava.mtv.fi -channel!axess.se -channel!axntv.de -channel!bandit.se -channel!bangutv.com -channel!bbcentertainment.com -channel!bbcentertainment.dk -channel!bbcentertainment.fi -channel!bbcknowledge.com -channel!bbcknowledge.dk -channel!bbcknowledge.fi -channel!bbcworldnews.com -channel!beate-uhse.tv -channel!belsat.eu -channel!berl.rbb-online.de -channel!bfs.daserste.de -channel!bibeltv.de -channel!bliss.tv2.no -channel!blizztv.de -channel!bongusto.tv -channel!bornholm.p4.dr.dk -channel!br-alpha.daserste.de -channel!brandnew.mtv.de -channel!brazzerstveurope.com -channel!bundesliga1.sky.de -channel!bundesliga10.sky.de -channel!bundesliga2.sky.de -channel!bundesliga3.sky.de -channel!bundesliga4.sky.de -channel!bundesliga5.sky.de -channel!bundesliga6.sky.de -channel!bundesliga7.sky.de -channel!bundesliga8.sky.de -channel!bundesliga9.sky.de -channel!bundesligahd1.sky.de -channel!bundesligahd10.sky.de -channel!bundesligahd2.sky.de -channel!bundesligahd3.sky.de -channel!bundesligahd4.sky.de -channel!bundesligahd5.sky.de -channel!bundesligahd6.sky.de -channel!bundesligahd7.sky.de -channel!bundesligahd8.sky.de -channel!bundesligahd9.sky.de -channel!bundesligauhd.sky.de -channel!bw.swr.daserste.de -channel!canal24h.rtve.es -channel!canal9.dk -channel!canale5.mediaset.it -channel!cbsreality.tv -channel!ch.viva.tv -channel!charlie.tv2.dk -channel!cinema.sky.de -channel!cinemagic.disneychannel.de -channel!cinemahd.sky.de -channel!cinemax.hr -channel!cinemax.si -channel!cinemax2.cinemax.hr -channel!cinemax2.cinemax.si -channel!clan.rtve.es -channel!classic.uptown.dk -channel!classic.vh1.se -channel!classica.de -channel!classics.kabel1.de -channel!classicshd.kabel1.de -channel!comedy.sky.de -channel!comedycentral.at -channel!comedycentral.ch -channel!comedycentral.de -channel!comedycentral.dk -channel!comedycentral.no -channel!comedycentral.pl -channel!comedycentral.tv -channel!crimeandinvestigation.eu -channel!crimeandinvestigation.pl -channel!crimehd.rtl.de -channel!dance.mtv.no -channel!dance.mtv.se -channel!daserste.de -channel!de.eonline.com -channel!deluxemusic.tv -channel!deutschplus.dw.de -channel!discovery.de -channel!discovery.dk -channel!discovery.fi -channel!discovery.no -channel!discoverychannel.se -channel!disneychannel.com -channel!disneychannel.de -channel!disneychannel.dk -channel!disneychannel.fi -channel!disneychannel.no -channel!disneychannel.se -channel!disneyxd.boxer.se -channel!dk.eonline.com -channel!dk4.dk -channel!dmax.discovery.de -channel!dmaxhd.discovery.de -channel!docubox.spi.pl -channel!doku.kabel1.de -channel!domatv.hr -channel!dr1.dr.dk -channel!dr1hd.dr.dk -channel!dr2.dr.dk -channel!dr3.dr.dk -channel!dr3hd.dr.dk -channel!dw.de -channel!eins.sky.de -channel!einsextra.daserste.de -channel!einsfestival.daserste.de -channel!einshd.sky.de -channel!emotion.cmore.dk -channel!emotion.cmore.fi -channel!emotion.cmore.no -channel!emotion.cmore.se -channel!emotion.sky.de -channel!emotions.sat1.de -channel!en.discovery.dk -channel!en.discovery.no -channel!en.discoverychannel.se -channel!en.tlc.discovery.dk -channel!en.tlc.discovery.no -channel!en.tlc.discovery.se -channel!epicdrama.ee.viasatworld.com -channel!epicdrama.lt.viasatworld.com -channel!epicdrama.lv.viasatworld.com -channel!erox.spi.pl -channel!eroxxx.spi.pl -channel!esbjerg.p4.dr.dk -channel!esports.viasat.dk -channel!esports.viasat.fi -channel!esports.viasat.no -channel!esports.viasat.se -channel!etv1.err.ee -channel!etv2.err.ee -channel!etvplus.err.ee -channel!euronews.com -channel!europa.tve.es -channel!europe.bloomberg.com -channel!europe.bluehustler.com -channel!europe.cnbc.com -channel!europe.daringtv.com -channel!europe.hustlertv.com -channel!europe.playboytv.com -channel!europe.realitykings.com -channel!eurosport.de -channel!eurosport.dk -channel!eurosport.fi -channel!eurosport.it -channel!eurosport.no -channel!eurosport.pl -channel!eurosport.sbsdiscovery.no -channel!eurosport.sbstv.dk -channel!eurosport.se -channel!eurosport2.eurosport.it -channel!eurosport2.fi -channel!eurosport2.no -channel!eurosport2.pl -channel!eurosport2.se -channel!eurosporthd.sbsdiscovery.no -channel!explore.ee.viasatworld.com -channel!explore.hr.viasatworld.com -channel!explore.hu.viasatworld.com -channel!explore.lt.viasatworld.com -channel!explore.lv.viasatworld.com -channel!explore.viasat.pl -channel!explorehd.viasat.dk -channel!explorehd.viasat.fi -channel!explorehd.viasat.no -channel!explorehd.viasat.se -channel!explorer.viasat.dk -channel!explorer.viasat.fi -channel!explorer.viasat.no -channel!explorer.viasat.se -channel!extra.mediaset.it -channel!extremesports.com -channel!fakta.tv4.se -channel!family.cinema.sky.de -channel!family.comedycentral.pl -channel!family.film.viasat.dk -channel!family.film.viasat.fi -channel!family.film.viasat.no -channel!family.film.viasat.se -channel!familyhd.cinema.sky.de -channel!familyhd.film.viasat.se -channel!familytv.de -channel!fashionbox.spi.pl -channel!fastandfun.spi.pl -channel!fatstone.tv -channel!fem.no -channel!fem.yle.fi -channel!fightbox.spi.pl -channel!film.tv4.se -channel!film.viasat.dk -channel!film.viasat.fi -channel!film.viasat.no -channel!film.viasat.se -channel!filmboxbasic.spi.pl -channel!filmboxfamily.spi.pl -channel!filmboxhd.spi.pl -channel!filmboxplus.spi.pl -channel!filmboxpremium.spi.pl -channel!first.cmore.dk -channel!first.cmore.fi -channel!first.cmore.no -channel!first.cmore.se -channel!firsthd.cmore.se -channel!fixundfoxi.tv -channel!focus.mediaset.it -channel!fotball.viasat.no -channel!fotboll-hockey-kids.cmore.se -channel!fotboll.cmore.se -channel!fotboll.viasat.se -channel!fox.fi -channel!foxchannel.de -channel!foxtv.ee -channel!foxtv.lt -channel!foxtv.lv -channel!foxtv.no -channel!foxtv.se -channel!fr.france24.com -channel!france24.com -channel!fri.tv2.dk -channel!frii.sbs.fi -channel!frikanalen.tv -channel!fsf.fightsports.tv -channel!ft.dk -channel!ftv.com -channel!fuel.tv -channel!fun.prosieben.de -channel!fyn.p4.dr.dk -channel!ginx.tv -channel!god.tv -channel!gold.sat1.de -channel!goldhd.sat1.de -channel!golf.cmore.dk -channel!golf.cmore.fi -channel!golf.cmore.se -channel!golf.viasat.dk -channel!golf.viasat.fi -channel!golf.viasat.no -channel!golf.viasat.se -channel!gospel.tv -channel!guld.tv4.se -channel!h2.historytv.dk -channel!h2.historytv.eu -channel!h2.historytv.fi -channel!h2.historytv.no -channel!h2.historytv.pl -channel!h2.historytv.se -channel!hbo.hr -channel!hbo.si -channel!hbo2.hbo.hr -channel!hbo2.hbo.si -channel!hbo3.hbo.hr -channel!hbo3.hbo.si -channel!hd.13thstreet.de -channel!hd.3sat.de -channel!hd.6-eren.dk -channel!hd.animalplanet.discovery.dk -channel!hd.animalplanet.discovery.no -channel!hd.animalplanet.se -channel!hd.anixehd.tv -channel!hd.arte.de -channel!hd.bbcentertainment.com -channel!hd.bbcentertainment.dk -channel!hd.bbcentertainment.fi -channel!hd.bbcentertainment.no -channel!hd.bbcknowledge.com -channel!hd.bbcknowledge.dk -channel!hd.bbcknowledge.fi -channel!hd.bbcknowledge.no -channel!hd.bibeltv.de -channel!hd.comedycentral.de -channel!hd.comedycentral.no -channel!hd.daserste.de -channel!hd.deluxemusic.tv -channel!hd.discovery.de -channel!hd.discovery.dk -channel!hd.discovery.eu -channel!hd.discovery.fi -channel!hd.discovery.no -channel!hd.discoverychannel.se -channel!hd.disneychannel.de -channel!hd.disneychannel.dk -channel!hd.disneychannel.fi -channel!hd.disneychannel.no -channel!hd.disneychannel.se -channel!hd.eurosport.de -channel!hd.eurosport.se -channel!hd.eurosport2.se -channel!hd.fem.no -channel!hd.film.viasat.se -channel!hd.first.cmore.dk -channel!hd.first.cmore.fi -channel!hd.first.cmore.no -channel!hd.fotboll.cmore.se -channel!hd.foxchannel.de -channel!hd.foxtv.no -channel!hd.foxtv.se -channel!hd.ftv.com -channel!hd.h2.historytv.se -channel!hd.historytv.de -channel!hd.historytv.se -channel!hd.hits.cmore.dk -channel!hd.hits.cmore.fi -channel!hd.hits.cmore.no -channel!hd.kabel1.de -channel!hd.kanal5.dk -channel!hd.kanal5.se -channel!hd.kanal9.se -channel!hd.kinowelt.tv -channel!hd.max.no -channel!hd.mezzo.tv -channel!hd.mtv.de -channel!hd.mtv.se -channel!hd.n-tv.de -channel!hd.natgeo.de -channel!hd.natgeo.se -channel!hd.nick.de -channel!hd.nickelodeon.dk -channel!hd.nickelodeon.fi -channel!hd.nickelodeon.no -channel!hd.nickelodeon.se -channel!hd.prosieben.de -channel!hd.rtl.de -channel!hd.rtl2.de -channel!hd.sat1.de -channel!hd.series.cmore.dk -channel!hd.series.cmore.fi -channel!hd.series.cmore.no -channel!hd.servustv.com -channel!hd.servustv.de -channel!hd.sixx.de -channel!hd.spiegel-geschichte.tv -channel!hd.sport1.mtv.fi -channel!hd.sport2.mtv.fi -channel!hd.syfy.de -channel!hd.tele5.de -channel!hd.tlcsverige.se -channel!hd.travelchanneltv.se -channel!hd.tv12.tv4.se -channel!hd.tv2.dk -channel!hd.tv2.no -channel!hd.tv3.no -channel!hd.tv3.se -channel!hd.tv4.se -channel!hd.tv6.se -channel!hd.tvnorge.no -channel!hd.tvp.pl -channel!hd.viasat4.no -channel!hd.viva.tv -channel!hd.vox.de -channel!hd.voxtv.no -channel!hd.zdf.de -channel!hdshowcase.discovery.dk -channel!hdshowcase.discovery.no -channel!hdshowcase.discoverychannel.com -channel!heaventv7.se -channel!heimatkanal.de -channel!herotv.fi -channel!historia.tvp.pl -channel!history.ee.viasatworld.com -channel!history.hr.viasatworld.com -channel!history.hu.viasatworld.com -channel!history.lt.viasatworld.com -channel!history.lv.viasatworld.com -channel!history.viasat.dk -channel!history.viasat.fi -channel!history.viasat.no -channel!history.viasat.pl -channel!history.viasat.se -channel!historyhd.viasat.dk -channel!historyhd.viasat.fi -channel!historyhd.viasat.no -channel!historyhd.viasat.se -channel!historytv.dk -channel!historytv.eu -channel!historytv.fi -channel!historytv.no -channel!historytv.pl -channel!historytv.se -channel!hits.cmore.dk -channel!hits.cmore.fi -channel!hits.cmore.no -channel!hits.cmore.se -channel!hits.film.viasat.dk -channel!hits.film.viasat.fi -channel!hits.film.viasat.no -channel!hits.film.viasat.se -channel!hits.mtv.no -channel!hits.mtv.se -channel!hits.sky.de -channel!hitshd.cmore.se -channel!hitshd.sky.de -channel!hockey.cmore.se -channel!hockey.viasat.fi -channel!hockey.viasat.no -channel!hockey.viasat.se -channel!hockeyfinland.viasat.fi -channel!hockeyhd.cmore.se -channel!hope-channel.de -channel!horseandcountry.tv -channel!hr.daserste.de -channel!hr1.hrt.hr -channel!hr2.hrt.hr -channel!hr3.hrt.hr -channel!hrhd.daserste.de -channel!htv1.hrt.hr -channel!htv2.hrt.hr -channel!htv3.hrt.hr -channel!htv4.hrt.hr -channel!humor.tv2.no -channel!humorhd.tv2.no -channel!info.tvp.pl -channel!infokanal.zdf.de -channel!int.kinopolska.pl -channel!int.kinopolskamuzyka.pl -channel!international.rt.com -channel!international.skynews.com -channel!investigation.discovery.dk -channel!investigation.discovery.it -channel!investigation.discovery.no -channel!investigation.discoverychannel.se -channel!iris.mediaset.it -channel!italia1.mediaset.it -channel!italia2.mediaset.it -channel!jaakiekkohd.viasat.fi -channel!jalkapallohd.viasat.fi -channel!jim.nelonen.fi -channel!joi.mediaset.it -channel!jr.disneychannel.de -channel!jr.nick.com.pl -channel!jr.nick.de -channel!jukebox-tv.de -channel!junior.disney.se -channel!junior.disneychannel.com -channel!junior.disneychannel.dk -channel!junior.disneychannel.fi -channel!junior.disneychannel.no -channel!junior.tv -channel!juniori.mtv.fi -channel!k.dr.dk -channel!kabel1.de -channel!kanal10.no -channel!kanal10.se -channel!kanal11.ee -channel!kanal12.ee -channel!kanal2.ee -channel!kanal4.dk -channel!kanal5.dk -channel!kanal5.se -channel!kanal9.se -channel!kanalhovedstaden.dk -channel!kanals2.lv -channel!kika.daserste.de -channel!kikahd.daserste.de -channel!kinopolska.pl -channel!kinopolskamuzyka.pl -channel!kinowelt.tv -channel!kobenhavn.p4.dr.dk -channel!krimi.sky.de -channel!kultura.tvp.pl -channel!kunskapskanalen.svt.se -channel!kunskapskanalenhd.svt.se -channel!kutonen.fi -channel!la.dw.de -channel!la1.rtve.es -channel!la2.rtve.es -channel!la5.mediaset.it -channel!langbolge.dr.dk -channel!life.foxtv.ee -channel!life.foxtv.lt -channel!life.foxtv.lv -channel!lifestyletv.se -channel!lifetimetv.pl -channel!liv.nelonen.fi -channel!live.cmore.se -channel!live2.cmore.se -channel!live2hd.cmore.se -channel!live3.cmore.se -channel!live3hd.cmore.se -channel!live4.cmore.se -channel!live4hd.cmore.se -channel!live5.cmore.se -channel!live5hd.cmore.se -channel!livehd.cmore.se -channel!livehdhitshd.cmore.boxer.se -channel!livsstilhd.tv2.no -channel!lnt.lv -channel!lugnafavoriter.se -channel!luxe.tv -channel!marcopolo.de -channel!matkanalen.tv -channel!max.mtv.fi -channel!max.no -channel!max.tv3.dk -channel!maxx.prosieben.de -channel!maxxhd.prosieben.de -channel!mdr.daserste.de -channel!mdrhd.daserste.de -channel!mezzo.tv -channel!midtvest.p4.dr.dk -channel!mojamini.tv -channel!motor.viasat.no -channel!motor.viasat.se -channel!motorhd.viasat.se -channel!motorvision.de -channel!mtv.ch -channel!mtv.de -channel!mtv.dk -channel!mtv.no -channel!mtv.pl -channel!mtv.se -channel!mtv3.fi -channel!music.mtv.pl -channel!musictelevision.fi -channel!n-tv.de -channel!n24doku.de -channel!natgeo.de -channel!natgeo.dk -channel!natgeo.ee -channel!natgeo.fi -channel!natgeo.lt -channel!natgeo.lv -channel!natgeo.no -channel!natgeo.pl -channel!natgeo.se -channel!nature-crime.viasat.se -channel!nature-history.hr.viasatworld.com -channel!nature-history.hu.viasatworld.com -channel!nature-history.viasat.pl -channel!nature-playboy.sat.viasat.se -channel!nature.hr.viasatworld.com -channel!nature.hu.viasatworld.com -channel!nature.viasat.dk -channel!nature.viasat.no -channel!nature.viasat.pl -channel!naturecee.ee.viasatworld.com -channel!naturecee.lt.viasatworld.com -channel!naturecee.lv.viasatworld.com -channel!naturecrime.viasat.fi -channel!naturehd.viasat.dk -channel!naturehd.viasat.fi -channel!naturehd.viasat.no -channel!naturehd.viasat.se -channel!nauticalchannel.net -channel!ndr.daserste.de -channel!ndrhd.daserste.de -channel!nelonen.fi -channel!neo.zdf.de -channel!neohd.zdf.de -channel!neokika.zdfmobil.de -channel!news.tv2.dk -channel!news.tv2.no -channel!newshd.tv2.no -channel!nick.ch -channel!nick.com.pl -channel!nick.de -channel!nickdk-mtvhits.sat.viasat.dk -channel!nickelodeon.at -channel!nickelodeon.dk -channel!nickelodeon.fi -channel!nickelodeon.no -channel!nickelodeon.se -channel!nickjr.dk -channel!nickjr.fi -channel!nickjr.no -channel!nickjr.se -channel!nickjrse-vh1se.sat.viasat.se -channel!nickno-vh1classic.sat.viasat.no -channel!nitro.rtl.de -channel!nitrohd.rtl.de -channel!no.bbcentertainment.no -channel!no.bbcknowledge.no -channel!no.eonline.com -channel!nordjylland.p4.dr.dk -channel!nostalgie.sky.de -channel!novatv.hr -channel!nrk1.nrk.no -channel!nrk1hd.nrk.no -channel!nrk2.nrk.no -channel!nrk2hd.nrk.no -channel!nrk3.nrk.no -channel!nrk3super.nrk.no -channel!nrk3superhd.nrk.no -channel!ok-kl.de -channel!ok-nahetv.de -channel!ok-weinstrasse.de -channel!ok-worms.de -channel!ok4.tv -channel!ok54.de -channel!oktv-lu.de -channel!oktv-mainz.de -channel!oktv-suedwestpfalz.de -channel!okv.se -channel!one.tv3sport.dk -channel!onehd.tv3sport.dk -channel!orf1.orf.at -channel!orf2.orf.at -channel!orf3.orf.at -channel!ostjylland.p4.dr.dk -channel!outdoorchannel.com -channel!outtv.se -channel!p1.cinema.sky.de -channel!p1.dr.dk -channel!p1.sr.se -channel!p1hd.cinema.sky.de -channel!p2.dr.dk -channel!p2.sr.se -channel!p24.cinema.sky.de -channel!p24hd.cinema.sky.de -channel!p3.dr.dk -channel!p3.sr.se -channel!p4gbg.sr.se -channel!p4malm.sr.se -channel!p4sth.sr.se -channel!p5.dr.dk -channel!p5sthlm.sr.se -channel!p6.dr.dk -channel!p7.dr.dk -channel!p8.dr.dk -channel!paramountchannel.pl -channel!passion.de -channel!people.natgeo.dk -channel!phoenix.daserste.de -channel!phoenixhd.daserste.de -channel!pl1.tv2.no -channel!pl2.tv2.no -channel!pl3.tv2.no -channel!planet-tv.de -channel!plus.toggo.de -channel!polonia.tvp.pl -channel!premium.tv1000.ee -channel!premium.tv1000.lt -channel!premium.tv1000.lv -channel!premiumactionhd.mediaset.it -channel!premiumcalcio1.mediaset.it -channel!premiumcalcio2.mediaset.it -channel!premiumcinema2hd.mediaset.it -channel!premiumcinemacomedy.mediaset.it -channel!premiumcinemaemotion.mediaset.it -channel!premiumcinemaenergyhd.mediaset.it -channel!premiumcinemahd.mediaset.it -channel!premiumcrimehd.mediaset.it -channel!premiumsport.mediaset.it -channel!premiumsport2.mediaset.it -channel!premiumstories.mediaset.it -channel!prosieben.de -channel!protv.ro -channel!puls.tv3.dk -channel!puls4.at -channel!puls8.ch -channel!radio1.srf.ch -channel!radio2.srf.ch -channel!radio3.srf.ch -channel!radio4.srf.ch -channel!radiom.srf.ch -channel!ramasjang.dr.dk -channel!rbb.daserste.de -channel!rbb.rbb-online.de -channel!rbbberl.rbb-online.de -channel!rbbbra.rbb-online.de -channel!rbbhd.daserste.de -channel!rck-tv.de -channel!retequattro.mediaset.it -channel!rheinmaintv.de -channel!rictv.de -channel!rixfm.se -channel!rocks.mtv.no -channel!rocks.mtv.se -channel!romance-tv.de -channel!rozrywka.tvp.pl -channel!rp.swr.daserste.de -channel!rtl.de -channel!rtl2.de -channel!rtlplus.de -channel!rts1.rts.ch -channel!rts2.rts.ch -channel!russia.rt.com -channel!s1tv.ch -channel!sat1.de -channel!science.discovery.dk -channel!science.discovery.fi -channel!science.discovery.no -channel!science.discoverychannel.com -channel!sd.anixehd.tv -channel!se.eonline.com -channel!select.sky.de -channel!selecthd.sky.de -channel!seriale.tvp.pl -channel!series.cmore.dk -channel!series.cmore.fi -channel!series.cmore.no -channel!series.cmore.se -channel!series.viasat.dk -channel!series.viasat.no -channel!series.viasat.se -channel!serieshd.cmore.se -channel!servustv.com -channel!servustv.de -channel!sf-kanalen.cmore.dk -channel!sf-kanalen.cmore.no -channel!sf-kanalen.cmore.se -channel!sf1.srf.ch -channel!sf2.srf.ch -channel!sfi.srf.ch -channel!sfkanalen.cmore.fi -channel!sixx.de -channel!sjaelland.p4.dr.dk -channel!sjuan.se -channel!sonyentertainment.tv -channel!spiegel-geschichte.tv -channel!sport-filmhd.cmore.se -channel!sport-sf.cmore.se -channel!sport.cmore.se -channel!sport.tv2.dk -channel!sport.tv2.no -channel!sport.tv3.se -channel!sport.tvp.pl -channel!sport.viasat.fi -channel!sport.viasat.no -channel!sport.viasat.se -channel!sport1.mtv.fi -channel!sport1.sky.de -channel!sport10.sky.de -channel!sport11.sky.de -channel!sport2.mtv.fi -channel!sport2.sky.de -channel!sport3.sky.de -channel!sport4.sky.de -channel!sport5.sky.de -channel!sport6.sky.de -channel!sport7.sky.de -channel!sport8.sky.de -channel!sport9.sky.de -channel!sportaustria.sky.de -channel!sportbaltic.viasat.ee -channel!sportbaltic.viasat.lt -channel!sportbaltic.viasat.lv -channel!sportdigital.tv -channel!sporthd.cmore.se -channel!sporthd.tv2.dk -channel!sporthd.tv2.no -channel!sporthd.tv3.se -channel!sporthd1.sky.de -channel!sporthd10.sky.de -channel!sporthd11.sky.de -channel!sporthd2.sky.de -channel!sporthd3.sky.de -channel!sporthd4.sky.de -channel!sporthd5.sky.de -channel!sporthd6.sky.de -channel!sporthd7.sky.de -channel!sporthd8.sky.de -channel!sporthd9.sky.de -channel!sportkanalen.se -channel!sportnews.sky.de -channel!sportnewshd.sky.de -channel!sportplus.orf.at -channel!sportpremium.viasat.fi -channel!sportpremium.viasat.se -channel!sportuhd.sky.de -channel!sr.swr.daserste.de -channel!srifm.sr.se -channel!studiouniversal.it -channel!sub.fi -channel!super.rtl.de -channel!superhd.rtl.de -channel!supertv.nrk.no -channel!svt1.svt.dev -channel=svt1.svt.se -channel!svt1hd.svt.se -channel=svt2.svt.se -channel!svt24.svt.se -channel!svt24hd.svt.se -channel!svt2hd.svt.se -channel!svtb-svt24.svt.se -channel!svtb.svt.se -channel!svtbhd.svt.se -channel!syd.p4.dr.dk -channel!syfy.de -channel!tele5.de -channel!teledeporte.rtve.es -channel!tlc.discovery.de -channel!tlc.discovery.dk -channel!tlc.discovery.eu -channel!tlc.discovery.no -channel!tlc.discoverynetworks.fi -channel!tlchd.discovery.no -channel!tlcsverige.se -channel!toons.nick.com.pl -channel!toons.nick.de -channel!toons.nickelodeon.dk -channel!toons.nickelodeon.fi -channel!toons.nickelodeon.no -channel!toons.nickelodeon.se -channel!topcrime.mediaset.it -channel!travelchanneltv.eu -channel!travelchanneltv.se -channel!trekanten.p4.dr.dk -channel!tv1.yle.dev -channel!tv1.yle.fi -channel!tv10.se -channel!tv1000.ee.viasatworld.com -channel!tv1000.lt.viasatworld.com -channel!tv1000.lv.viasatworld.com -channel!tv1000action.ee.viasatworld.com -channel!tv1000action.lt.viasatworld.com -channel!tv1000action.lv.viasatworld.com -channel!tv1000balkans.hr.viasatworld.com -channel!tv1000comedyhd.ee.viasatworld.com -channel!tv1000comedyhd.lt.viasatworld.com -channel!tv1000comedyhd.lv.viasatworld.com -channel!tv1000kino.ee.viasatworld.com -channel!tv1000kino.lt.viasatworld.com -channel!tv1000kino.lv.viasatworld.com -channel!tv11.sbstv.se -channel!tv11hd.sbstv.se -channel!tv12.tv4.se -channel!tv2.dk -channel!tv2.no -channel!tv2.yle.fi -channel!tv24.ch -channel!tv25.ch -channel!tv2bornholm.dk -channel!tv2fyn.dk -channel!tv2lorry.dk -channel!tv2nord.dk -channel!tv2oj.dk -channel!tv3.dk -channel!tv3.ee -channel!tv3.lt -channel!tv3.lv -channel!tv3.no -channel!tv3.se -channel!tv3plus.dk -channel!tv4.se -channel!tv4fakta.boxer.se -channel!tv4film.boxer.se -channel!tv5.fi -channel!tv5monde.org -channel!tv6.ee -channel!tv6.lt -channel!tv6.lv -channel!tv6.se -channel!tv6norge.no -channel!tv7.fi -channel!tv7plus.fi -channel!tv8.lt -channel!tv8.se -channel!tveast.dk -channel!tvfinland.yle.fi -channel!tvmidtvest.dk -channel!tvnorge.no -channel!tvp1.tvp.pl -channel!tvp2.tvp.pl -channel!tvsyd.dk -channel!ultra.dr.dk -channel!ultrahd.viasat.dk -channel!ultrahd.viasat.fi -channel!ultrahd.viasat.no -channel!ultrahd.viasat.se -channel!universalchannel.de -channel!urbanint.trace.tv -channel!vh1.eu -channel!vh1.fi -channel!vh1.mtv.dk -channel!vh1.mtv.pl -channel!vh1.no -channel!vh1.se -channel!viasat3.hu -channel!viasat4.no -channel!viasat6.hu -channel!visionsverige.com -channel!visjonnorge.com -channel!viva-tv.pl -channel!viva.tv -channel!vox.de -channel!voxtv.no -channel!wdr.daserste.de -channel!wdrhd.daserste.de -channel!welt.de -channel!wild.natgeo.de -channel!wild.natgeo.dk -channel!wild.natgeo.fi -channel!wild.natgeo.no -channel!wild.natgeo.pl -channel!wild.natgeo.se -channel!wildhd.natgeo.de -channel!wildhd.natgeo.se -channel!world.discovery.dk -channel!world.discovery.eu -channel!world.discovery.no -channel!world.discoveryworld.se -channel!world.kbs.co.kr -channel!xd.disneychannel.com -channel!xd.disneychannel.de -channel!xd.disneychannel.dk -channel!xd.disneychannel.fi -channel!xd.disneychannel.no -channel!xd.disneychannel.se -channel!xee.fox.com -channel!xite.tv -channel!zdf.de -channel!zebra.tv2.no -channel!zebrahd.tv2.no -channel!zulu.tv2.dk diff -Nru xmltv-0.6.1/grab/se_tvzon/tv_grab_se_tvzon.PL xmltv-0.6.3/grab/se_tvzon/tv_grab_se_tvzon.PL --- xmltv-0.6.1/grab/se_tvzon/tv_grab_se_tvzon.PL 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/se_tvzon/tv_grab_se_tvzon.PL 1970-01-01 00:00:00.000000000 +0000 @@ -1,23 +0,0 @@ -# Generate tv_grab_se_tvzon from tv_grab_se_swedb.in. -# - -use strict; - -use IO::File; -my $out = shift @ARGV; die "no output file given" if not defined $out; -my $in = 'grab/se_swedb/tv_grab_se_swedb.in'; -my $out_fh = new IO::File "> $out" or die "cannot write to $out: $!"; -my $in_fh = new IO::File "< $in" or die "cannot read $in: $!"; -my $seen = 0; -while (<$in_fh>) { - s/\@\@name/tv_grab_se_tvzon/; - s/\@\@nspc/ /; - s/\@\@country/Sweden/; - s/\@\@desc/Sweden (DEPRECATED, TVZon)/; - s%\@\@url%http://xmltv.xmltv.se/channels.xml.gz%; - s%\@\@site%http://xmltv.xmltv.se/%; - print $out_fh $_; -} -close $out_fh or die "cannot close $out: $!"; -close $in_fh or die "cannot close $in: $!"; - diff -Nru xmltv-0.6.1/grab/uk_tvguide/tv_grab_uk_tvguide xmltv-0.6.3/grab/uk_tvguide/tv_grab_uk_tvguide --- xmltv-0.6.1/grab/uk_tvguide/tv_grab_uk_tvguide 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/uk_tvguide/tv_grab_uk_tvguide 2020-09-07 15:02:53.000000000 +0000 @@ -54,9 +54,9 @@ my $GRABBER_NAME = 'tv_grab_uk_tvguide'; my $GRABBER_DESC = 'UK - TV Guide (tvguide.co.uk)'; my $GRABBER_URL = 'http://wiki.xmltv.org/index.php/XMLTVProject'; -my $ROOT_URL = 'http://my.tvguide.co.uk/'; +my $ROOT_URL = 'https://www.tvguide.co.uk/'; my $SOURCE_NAME = 'TV Guide UK'; -my $SOURCE_URL = 'http://my.tvguide.co.uk/'; +my $SOURCE_URL = 'https://www.tvguide.co.uk/'; # my $generator_info_name = $GRABBER_NAME; my $generator_info_url = $GRABBER_URL; @@ -180,7 +180,7 @@ # But this works too: # http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime=3/18/2013 - my $baseurl = $ROOT_URL.'channellisting.asp'; + my $baseurl = $ROOT_URL.'channellistings.asp'; # Now grab listings for each channel on each day, according to the options in $opt # @@ -208,7 +208,7 @@ # tvguide website can be very slow - try to avoid barfing when no response if (!defined $channelname) { - print STDERR "Unable to retrieve web page for $channel_id \n"; + warning "Unable to retrieve web page for $channel_id"; next; } @@ -358,7 +358,7 @@ # Unfortunately the div with the date doesn't have any safe identifier. There are several ways we could remove the # cruft from the container but the following, although clunky, is probably the safest - my ($dt, $h, $i, $a, $h2, $i2, $a2) = $lhs->as_text =~ /((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun|Christmas\s(?:Eve|Day)|Boxing\sDay|New\sYear).*?)(\d*):(\d*)(am|pm)(?:-(\d*):(\d*)(am|pm))?/; + my ($dt, $h, $i, $a, $h2, $i2, $a2) = $lhs->as_text =~ /((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun|Christmas\s(?:Eve|Day)|Boxing\sDay|New\sYears\s(?:Eve|Day))[\s<].*?)(\d*):(\d*)(am|pm)(?:-(\d*):(\d*)(am|pm))?/; # print STDERR $dt."\n"; if ($dt && $dt !~ /\D\D\D\s\d\d?\s\D\D\D/) { @@ -392,6 +392,7 @@ $h2 -= 12 if $a2 eq 'am' && $h2 == 12; $showtime->set(hour => $h, minute => $i, second => 0); $prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z"); + my $showtime_ = $showtime->clone; if (defined $h2 && $h2 >= 0) { $showtime->add (days => 1) if $h2 < $h; # see note above re errors with GMT/BST transition @@ -399,7 +400,15 @@ $showtime->set(hour => $h2, minute => $i2, second => 0); $prog{'stop'} = $showtime->strftime("%Y%m%d%H%M%S %z"); } or do { # catch - # no output prog 'stop' time + # let's see if we can get a duration + my ($durh, $durm) = $lhs->as_text =~ /\((?:(\d*)\shours?)?\s?(?:(\d*)\sminutes?)?\)/; + if (defined $durh || defined $durm) { + $durh = 0 if !defined $durh; $durm = 0 if !defined $durm; + $showtime_->set_time_zone('UTC')->add( hours => $durh, minutes => $durm )->set_time_zone('Europe/London'); + $prog{'stop'} = $showtime_->strftime("%Y%m%d%H%M%S %z"); + } else { + # no output prog 'stop' time + } } } else { # no output prog 'stop' time @@ -411,7 +420,7 @@ } # end showdetail - $showdetail->delete(); + $showdetail->delete() if $showdetail; } @@ -741,7 +750,7 @@ # Fetch channels via a dummy call to BBC1 listings # http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime= - my $channel_list = $ROOT_URL.'channellisting.asp?ch=86&cTime='; + my $channel_list = $ROOT_URL.'channellistings.asp?ch=74&cTime='; my $result; my $channels = {}; @@ -770,7 +779,7 @@ foreach my $channel (@channels) { if ($channel->as_text) { my ($id) = $channel->attr('value'); - my ($url) = 'channellisting.asp?ch=' . $channel->attr('value'); + my ($url) = 'channellistings.asp?ch=' . $channel->attr('value'); my ($name) = $channel->as_text; $channels->{"$id"} = { diff -Nru xmltv-0.6.1/grab/zz_sdjson/tv_grab_zz_sdjson xmltv-0.6.3/grab/zz_sdjson/tv_grab_zz_sdjson --- xmltv-0.6.1/grab/zz_sdjson/tv_grab_zz_sdjson 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/zz_sdjson/tv_grab_zz_sdjson 2020-09-07 15:02:53.000000000 +0000 @@ -1327,11 +1327,13 @@ } # The xmltv docs state this field is "When and where the programme was last shown". -# However mythtv expects the original air date to be in this field. +# Programs that are marked as new by Schedules Direct can not have a XMLTV previously_shown. sub get_program_previously_shown { - my ($details) = @_; + my ($program, $details) = @_; my %previously_shown; + return undef if(get_program_new($program)); + my $date = $details->{'originalAirDate'}; if($date) { my $dt = parse_original_airdate($date); @@ -1447,10 +1449,10 @@ 'episode-num' => get_program_episode($program, $details), 'video' => get_program_video($program), 'audio' => get_program_audio($program), - 'previously-shown' => get_program_previously_shown($details), + 'previously-shown' => get_program_previously_shown($program, $details), 'premiere' => get_program_premiere($program), # 'last-chance' => undef, - 'new' => get_program_new($program), +# 'new' => undef, 'subtitles' => get_program_subtitles($program), 'rating' => get_program_rating($program, $details), 'star-rating' => get_program_star_rating($details), diff -Nru xmltv-0.6.1/grab/zz_sdjson_sqlite/fixups.txt xmltv-0.6.3/grab/zz_sdjson_sqlite/fixups.txt --- xmltv-0.6.1/grab/zz_sdjson_sqlite/fixups.txt 1970-01-01 00:00:00.000000000 +0000 +++ xmltv-0.6.3/grab/zz_sdjson_sqlite/fixups.txt 2020-09-07 15:02:53.000000000 +0000 @@ -0,0 +1,60 @@ + +FIXUPS + +Some applications are known to not be compliant with the XMLTV specifications, +or have other peculiarities in the way they interact with the grabber output. +Users of those applications should request that the developers correct their +implementation or add the required additional functionality they desire and +then migrate to those fixed versions, but there are cases where that cannot +be accomplished quickly. To address this issue, this grabber will recognize +a request for fixups in an environment variable. + +Fixups are intended to be a temporary measure. It is imperative that users +work with the developers of their application to release an updated version, +and that one updates to that release. While it is a goal that a fixup be +supported for at least a year after the application has been identified as +being non-complaint with the XMLTV definition or that the application does +not provide the desired functionality, if code changes or refactor in this +grabber impact the ability to support a fixup, the fixup may be removed +sooner. + +NOTE: Requests to add fixups should include a patch or pull request. + +The environmental variable is TV_GRAB_TARGET_APPLICATION_FIXUPS and the +requested fixups are separated by a colon. + +Example usage: + + TV_GRAB_TARGET_APPLICATION_FIXUPS=NO_XMLTV_NS_TOTAL_SEASONS:NO_PREVIOUSLY_SHOWN_ZONE_OFFSET tv_grab_zz_sdjson_sqlite + +Currently implemented fixups: + + NO_XMLTV_NS_TOTAL_SEASONS + + Do not add in the total seasons value to season value in the xmltv_ns + episode numbering. + + Known apps: MythTV master before 585f509 (fixes/0.28 before e26a33c) + + NO_PREVIOUSLY_SHOWN_ZONE_OFFSET + + Do not add in the zulu offset for previously shown. + + Known apps: MythTV versions before ff5ab27 (legacy unsupported versions) + + NO_STATION_LOGOS + + Do not add the station logos/icons to the generated result. This is + (mostly?) useful when an individual has carefully curated a set of + logos and the application will replace them with the logos provided + by the xmltv provided values without further user interaction. + + Known apps: MythTV (feature request posted) + + NO_MULTIPLE_STATION_LOGOS + + Only return the first station logo/icon. This is mostly useful + when the app chooses the last, rather than the first, logo when + presented with more than one logo. + + Known apps: MythTV before 96e307a (legacy unsupported versions) diff -Nru xmltv-0.6.1/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite xmltv-0.6.3/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite --- xmltv-0.6.1/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite 2020-09-07 15:02:53.000000000 +0000 @@ -44,6 +44,41 @@ # # Version history: # +# 2020/06/21 - 1.101 - rename scaledownload to scale-download +# 2020/06/20 - 1.100 - add support for --scaledownload +# 2020/06/12 - 1.99 - include programID in metadata +# 2020/05/18 - 1.98 - support ordering of station logos +# 2020/05/17 - 1.97 - explicitly specify stable sort +# 2020/05/09 - 1.96 - improve passwordhash option handling +# 2020/05/08 - 1.95 - improve metadata names based on feedback +# 2020/05/08 - 1.94 - extend metadata with schedules direct values +# 2020/05/08 - 1.93 - refactor obtainStationsSchedules +# 2020/05/05 - 1.92 - error checking and handling improvements +# 2020/05/05 - 1.91 - increase potential grabber concurrency phase 3 +# 2020/04/27 - 1.90 - increase potential grabber concurrency phase 2 +# 2020/04/26 - 1.89 - increase potential grabber concurrency phase 1 +# 2020/04/23 - 1.88 - reorganize database open/validation +# 2020/04/13 - 1.87 - additional validation of returned data +# 2020/04/10 - 1.86 - refactor obtainStationsSchedulesHash +# 2020/04/07 - 1.85 - partially revert location removal +# 2020/04/07 - 1.84 - fix for manage-lineups channel selection +# 2020/04/06 - 1.83 - fix for manage-lineups with no database +# 2020/04/05 - 1.82 - change lineup to lineupID for obtainLineups +# 2020/04/04 - 1.81 - refactor/rename obtainHeadends +# 2020/04/02 - 1.80 - remove location from lineup displays +# 2020/04/01 - 1.79 - do not validate postal code via regex +# 2020/03/30 - 1.78 - robustify token reuse validation +# 2020/03/28 - 1.77 - use obtainLineups where appropriate +# 2020/03/28 - 1.76 - supplement lineup data with status data +# 2020/03/27 - 1.75 - remove legacy (20131021) api name +# 2020/03/23 - 1.74 - refactor obtainLineups to return lineup array +# 2020/03/22 - 1.73 - allow Schedules Direct endpoint redirects +# 2020/03/22 - 1.72 - reuse existing token when possible +# 2020/03/18 - 1.71 - handle obtainAvailable undef +# 2020/03/18 - 1.70 - minor whitespace cleanup +# 2020/03/17 - 1.69 - stable output order +# 2020/03/17 - 1.68 - return all icons for channels unless fixup +# 2019/11/08 - 1.67 - handle no-download for list-lineups # 2018/12/21 - 1.66 - default 3rdparty metadata in configure to disabled # 2018/12/17 - 1.65 - clean up whitespace and duplicate lines # 2018/12/16 - 1.64 - add support for gracenote rating body advisories @@ -127,11 +162,12 @@ STDERR->autoflush(1); # Autoflush STDERR -use XMLTV 0.005067; +use XMLTV; use XMLTV::Options qw/ParseOptions/; use XMLTV::Configure::Writer; use XMLTV::Configure qw/LoadConfig SaveConfig/; use XMLTV::Ask; +use Getopt::Long; use XML::Writer; use Encode qw/decode encode/; use JSON; @@ -152,11 +188,12 @@ use DBD::SQLite; use Scalar::Util qw/looks_like_number/; use Data::Dumper; +use sort 'stable'; my $RFC2838_COMPLIANT = 1; # RFC2838 compliant station ids, which makes XMLTV # validate even though the docs say "SHOULD" not "MUST" -my $SCRIPT_VERSION = '$Id: tv_grab_zz_sdjson_sqlite,v 1.66 2018/12/21 11:35:00 gtb Exp ed $'; +my $SCRIPT_VERSION = '$Id: tv_grab_zz_sdjson_sqlite,v 1.101 2020/06/21 20:30:00 gtb Exp ed $'; my $SCRIPT_URL = 'https://github.com/garybuhrmaster/tv_grab_zz_sdjson_sqlite'; my $SCRIPT_NAME = basename("$0"); my $SCRIPT_NAME_DIR = dirname("$0"); @@ -175,6 +212,10 @@ my $SD_SCHEDULE_CHUNK = 1000; # Request stations schedules in chunk sizes my $SD_PROGRAM_CHUNK = 4000; # Request program data in chunk sizes +my $SD_SCHEDULE_HASH_CHUNK_MAX = 5000; # Schedules Direct max request size +my $SD_SCHEDULE_CHUNK_MAX = 5000; # Schedules Direct max request size +my $SD_PROGRAM_CHUNK_MAX = 5000; # Schedules Direct max request size + my $JSON = JSON->new()->shrink(1)->utf8(1); my $SD = SchedulesDirect->new(); @@ -200,6 +241,15 @@ my $opt; my $conf; +# +# We attempt to pick off the --passwordhash option due to +# the XMLTV ParseOptions not allowing extra_options to be +# processed in the configure stage. +# +Getopt::Long::Configure("pass_through"); +GetOptions('passwordhash=s' => \$passwordHash); +Getopt::Long::Configure("no_pass_through"); + ( $opt, $conf ) = ParseOptions ( { @@ -213,14 +263,13 @@ preferredmethod => 'allatonce', version => "$SCRIPT_VERSION", description => 'Multinational (Schedules Direct JSON web services with SQLite DB)', - extra_options => [qw/manage-lineups force-download download-only no-download passwordhash=s/], + extra_options => [qw/manage-lineups force-download download-only no-download passwordhash=s scale-download=f/], defaults => { days => 30 }, } ); $debug = $opt->{'debug'}; $quiet = $opt->{'quiet'}; -$passwordHash = $opt->{'passwordhash'}; $SD->Debug(1) if ($debug); @@ -263,6 +312,12 @@ exit(1); } +if (defined($opt->{'scale-download'}) && looks_like_number($opt->{'scale-download'})) + { + $SD_SCHEDULE_HASH_CHUNK = min($SD_SCHEDULE_HASH_CHUNK_MAX, max(1, int($SD_SCHEDULE_HASH_CHUNK * $opt->{'scale-download'}))); + $SD_SCHEDULE_CHUNK = min($SD_SCHEDULE_CHUNK_MAX, max(1, int($SD_SCHEDULE_CHUNK * $opt->{'scale-download'}))); + $SD_PROGRAM_CHUNK = min($SD_PROGRAM_CHUNK_MAX, max(1, int($SD_PROGRAM_CHUNK * $opt->{'scale-download'}))); + } if (!defined(eval {require JSON::XS})) { @@ -317,6 +372,8 @@ if (!$download) { lineupValidate($conf->{'lineup'}); + my $token = DB_settingsGet('token'); + $SD->Token($token) if (defined($token)); goto skipDownload; } @@ -417,17 +474,27 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } my $fetchStationSchedulesRequired = $sth->fetchrow_array() || 0;; +if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } $sth->finish(); +$DBH->commit(); + +undef $sth; + if ((!$fetchStationSchedulesRequired) && (!$fetchLineupRequired)) { print (STDERR " not downloading station schedule hashes (data current)\n") if (!$quiet); @@ -438,14 +505,6 @@ # Obtain the current schedule hash values for our # lineup stations and feed to our DB # - # Note that there is no (substantial) advantage in - # requesting only the days we will be processing - # as tests have shown that Schedules Direct takes - # about the same time to return all vs just one, - # and it complicates matters to request ranges - # and deal with potential errors due to out of - # range issues. - # $sql = 'select distinct stations.station from stations as stations where stations.station in (select distinct channels.station from channels as channels where channels.lineup in ( ' . join(', ', ('?') x scalar(@{$conf->{'lineup'}})) . ' ) and channels.selected = 1)'; $sth = $DBH->prepare_cached($sql); @@ -463,29 +522,29 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); my $stationsSchedulesHashList = $sth->fetchall_arrayref([0]); - - $sth->finish(); - - print (STDERR " downloading station schedule hashes for " . scalar(@{$stationsSchedulesHashList}) . " stations\n") if (!$quiet); - - $sql = "replace into stations_schedules_hash (station, day, hash, details) values ( ?, ?, ?, ?)"; - $sth = $DBH->prepare_cached($sql); - if (!defined($sth)) + if ($sth->err()) { - print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } + $DBH->commit(); + + undef $sth; + + print (STDERR " downloading station schedule hashes for " . scalar(@{$stationsSchedulesHashList}) . " stations\n") if (!$quiet); + my $stationsSchedulesHashIter; $stationsSchedulesHashIter = natatime $SD_SCHEDULE_HASH_CHUNK, @{$stationsSchedulesHashList}; while(my @chunk = $stationsSchedulesHashIter->()) @@ -508,38 +567,56 @@ exit(1); } - if (ref($r) ne 'HASH') + if (ref($r) ne 'ARRAY') { print (STDERR "Unexpected return data type " . ref($r) . " when obtaining station schedules hashes.\n"); exit(1); } - foreach my $station(keys %{$r}) + $sql = "replace into stations_schedules_hash (station, day, hash, details) values ( ?, ?, ?, ?)"; + + $sth = $DBH->prepare_cached($sql); + if (!defined($sth)) + { + print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + exit(1); + } + + foreach my $e(@{$r}) { - if (ref($r->{$station}) ne 'HASH') + if (ref($e) ne 'HASH') { - # print (STDERR "Unexpected return data type " . ref($r->{$station}) . " for station $station while obtaining station schedules hashes\n"); + # print (STDERR "Unexpected return data type " . ref($e) . " while iterating station schedules hashes\n"); next; } - foreach my $day(keys %{$r->{$station}}) + if ((!defined($e->{'stationID'})) || + (!defined($e->{'date'})) || + ((substr($e->{'date'}, 0, 10)) !~ /^\d{4}-\d{2}-\d{2}$/) || + (!defined($e->{'MD5'}))) { - my $s = $r->{$station}->{$day}; - my $hash = $s->{'md5'} || ''; - my $details = $JSON->utf8->encode($s); - $sth->bind_param( 1, $station, SQL_VARCHAR ); - $sth->bind_param( 2, $day, SQL_DATE ); - $sth->bind_param( 3, $hash, SQL_VARCHAR ); - $sth->bind_param( 4, $details, SQL_VARCHAR ); - $sth->execute(); - if ($sth->err) - { - print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1); - } + # print (STDERR "Station, date, or hash not provided while iterating station schedules hashes\n"); + next; + } + my $station = $e->{'stationID'}; + my $date = substr($e->{'date'}, 0, 10); + my $hash = $e->{'MD5'}; + my $details = $JSON->utf8->encode($e); + $sth->bind_param( 1, $station, SQL_VARCHAR ); + $sth->bind_param( 2, $date, SQL_DATE ); + $sth->bind_param( 3, $hash, SQL_VARCHAR ); + $sth->bind_param( 4, $details, SQL_VARCHAR ); + $sth->execute(); + if ($sth->err) + { + print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); } } $DBH->commit(); + + undef $sth; } # @@ -564,9 +641,12 @@ if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $DBH->commit(); + + undef $sth; } # @@ -626,6 +706,7 @@ if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } @@ -633,8 +714,16 @@ $sth->bind_col( 2, undef, SQL_DATE ); my $stationsSchedulesList = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; if (scalar(@{$stationsSchedulesList}) == 0) { @@ -649,29 +738,6 @@ sleep(min(30, (10 * $retry))); - $sql1 = "delete from schedules where station = ? and day = ?"; - $sql2 = "replace into schedules (station, day, starttime, duration, program, program_hash, details) values (?, ?, ?, ?, ?, ?, ?)"; - $sql3 = "replace into schedules_hash (station, day, hash) values (?, ?, ?)"; - - $sth1 = $DBH->prepare_cached($sql1); - if (!defined($sth1)) - { - print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); - exit(1); - } - $sth2 = $DBH->prepare_cached($sql2); - if (!defined($sth2)) - { - print (STDERR "Unexpected error when preparing statement ($sql2): " . $DBH->errstr . "\n"); - exit(1); - } - $sth3 = $DBH->prepare_cached($sql3); - if (!defined($sth3)) - { - print (STDERR "Unexpected error when preparing statement ($sql3): " . $DBH->errstr . "\n"); - exit(1); - } - my $schedulesIter; $schedulesIter = natatime $SD_SCHEDULE_CHUNK, @{$stationsSchedulesList}; while(my @chunk = $schedulesIter->()) @@ -710,10 +776,34 @@ next; } + $sql1 = "delete from schedules where station = ? and day = ?"; + $sql2 = "replace into schedules (station, day, starttime, duration, program, program_hash, details) values (?, ?, ?, ?, ?, ?, ?)"; + $sql3 = "replace into schedules_hash (station, day, hash) values (?, ?, ?)"; + + $sth1 = $DBH->prepare_cached($sql1); + if (!defined($sth1)) + { + print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); + exit(1); + } + $sth2 = $DBH->prepare_cached($sql2); + if (!defined($sth2)) + { + print (STDERR "Unexpected error when preparing statement ($sql2): " . $DBH->errstr . "\n"); + exit(1); + } + $sth3 = $DBH->prepare_cached($sql3); + if (!defined($sth3)) + { + print (STDERR "Unexpected error when preparing statement ($sql3): " . $DBH->errstr . "\n"); + exit(1); + } + foreach my $sched(@{$r}) { - my $hash; + my $hash = $sched->{'MD5'}; my $dayDateTime; + $dayDateTime = DateTime::Format::ISO8601->parse_datetime($sched->{'date'}) if (defined($sched->{'date'})); my $sID = $sched->{'stationID'}; my $code = $sched->{'code'} || 0; if ($code != 0) @@ -724,15 +814,6 @@ } next; } - my $meta = $sched->{'metadata'}; - if (defined($meta)) - { - $hash = $meta->{'md5'}; - if (defined($meta->{'startDate'})) - { - $dayDateTime = DateTime::Format::ISO8601->parse_datetime($meta->{'startDate'}); - } - } my $programs = $sched->{'programs'}; if ((!defined($hash)) || (!defined($dayDateTime)) || (!defined($programs))) { @@ -744,7 +825,8 @@ if ($sth1->err) { print (STDERR "Unexpected error when executing statement ($sql1): " . $sth1->errstr . "\n"); - exit(1); + $DBH->rollback(); + exit(1); } foreach my $program(@{$programs}) { @@ -755,7 +837,7 @@ my $details = $JSON->utf8->encode($program); if ((!defined($duration)) || (!defined($phash)) || (!defined($pID)) || (!defined($airDateTime))) { - print (STDERR "Unexpected parsing error in program (data malformed) in schedule for $sID on " . $meta->{'startDate'} . ", skipping\n") if (!$quiet); + print (STDERR "Unexpected parsing error in program (data malformed) in schedule for $sID on " . $sched->{'date'} . ", skipping\n") if (!$quiet); next; } my $starttime = DateTime::Format::ISO8601->parse_datetime($airDateTime); @@ -770,6 +852,7 @@ if ($sth2->err) { print (STDERR "Unexpected error when executing statement ($sql2): " . $sth2->errstr . "\n"); + $DBH->rollback(); exit(1); } } @@ -780,11 +863,16 @@ if ($sth3->err) { print (STDERR "Unexpected error when executing statement ($sql3): " . $sth3->errstr . "\n"); - exit(1); + $DBH->rollback(); + exit(1); } } $DBH->commit(); + + undef $sth1; + undef $sth2; + undef $sth3; } # We are done unless one (or more) entities indicate that the server queued the request @@ -843,18 +931,26 @@ $param++; $sth->execute(); - if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); my $programsList = $sth->fetchall_arrayref([0]); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; if (scalar(@{$programsList}) == 0) { @@ -869,15 +965,6 @@ sleep(min(30, (10 * $retry))); - $sql1 = "replace into programs (program, hash, details, program_supplemental, downloaded) values (?, ?, ?, ?, ?)"; - - $sth1 = $DBH->prepare_cached($sql1); - if (!defined($sth1)) - { - print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); - exit(1); - } - my $programsIter; $programsIter = natatime $SD_PROGRAM_CHUNK, @{$programsList}; while(my @chunk = $programsIter->()) @@ -913,6 +1000,15 @@ next; } + $sql1 = "replace into programs (program, hash, details, program_supplemental, downloaded) values (?, ?, ?, ?, ?)"; + + $sth1 = $DBH->prepare_cached($sql1); + if (!defined($sth1)) + { + print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); + exit(1); + } + foreach my $program(@{$r}) { my $pID = $program->{'programID'}; @@ -943,10 +1039,13 @@ if ($sth1->err) { print (STDERR "Unexpected error when executing statement ($sql1): " . $sth1->errstr . "\n"); + $DBH->rollback(); exit(1); } } $DBH->commit(); + + undef $sth1; } # We are done unless one (or more) entities indicate that the server queued the request @@ -998,18 +1097,26 @@ $param++; $sth->execute(); - if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } $sth->bind_col(1, undef, SQL_VARCHAR ); my $programsList = $sth->fetchall_arrayref([0]); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; if (scalar(@{$programsList}) == 0) { @@ -1024,15 +1131,6 @@ sleep(min(30, (10 * $retry))); - $sql1 = "replace into programs (program, hash, details, program_supplemental, downloaded) values (?, ?, ?, ?, ?)"; - - $sth1 = $DBH->prepare_cached($sql1); - if (!defined($sth1)) - { - print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); - exit(1); - } - my $programsIter; $programsIter = natatime $SD_PROGRAM_CHUNK, @{$programsList}; while(my @chunk = $programsIter->()) @@ -1070,6 +1168,15 @@ exit(1); } + $sql1 = "replace into programs (program, hash, details, program_supplemental, downloaded) values (?, ?, ?, ?, ?)"; + + $sth1 = $DBH->prepare_cached($sql1); + if (!defined($sth1)) + { + print (STDERR "Unexpected error when preparing statement ($sql1): " . $DBH->errstr . "\n"); + exit(1); + } + foreach my $program(@{$r}) { my $pID = $program->{'programID'}; @@ -1100,10 +1207,13 @@ if ($sth1->err) { print (STDERR "Unexpected error when executing statement ($sql1): " . $sth1->errstr . "\n"); + $DBH->rollback(); exit(1); } } $DBH->commit(); + + undef $sth1; } # We are done unless one (or more) entities indicate that the server queued the request @@ -1270,6 +1380,7 @@ 'source-info-url' => $SD_SITEURL ); my $channelsWritten = channelWriter($conf->{'lineup'}, $w); + print (STDERR " $channelsWritten channels processed\n") if (!$quiet); # @@ -1310,11 +1421,11 @@ $param++; $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -1729,15 +1840,6 @@ $w->dataElement('episode-num', $prodid, 'system' => 'dd_progid' ); } - #if (defined($programDetails->{'resourceID'})) - # { - # $w->dataElement('episode-num', "resourceid/$programDetails->{'resourceID'}", 'system' => 'schedulesdirect.org'); - # } - #elsif (defined($supplementalDetails->{'resourceID'})) - # { - # $w->dataElement('episode-num', "resourceid/$supplementalDetails->{'resourceID'}", 'system' => 'schedulesdirect.org'); - # } - # Season/Episode numbering is "special" as SHows and # EPisodes use slightly different interpretations of # the exact same terms. @@ -1838,6 +1940,7 @@ { $w->dataElement('episode-num', " $season . $episode . $part ", 'system' => 'xmltv_ns'); } + if (defined($TVDBepisodeID)) { if ((defined($conf->{'3rdparty-metadata'}->[0])) && @@ -1855,6 +1958,82 @@ } } + # + # Potentionally need to know if this is a new showing + # in the extra metadata section, and again for the + # previously shown determination + # + my $newShowing = 0; + $newShowing = $scheduleDetails->{'new'} if (defined($scheduleDetails->{'new'})); + + if ((defined($conf->{'3rdparty-metadata'}->[0])) && + ($conf->{'3rdparty-metadata'}->[0] eq 'enabled')) + { + $w->dataElement('episode-num', "programID/$pID", 'system' => 'schedulesdirect.org' ); + if (defined($programDetails->{'resourceID'})) + { + $w->dataElement('episode-num', "resourceID/$programDetails->{'resourceID'}", 'system' => 'schedulesdirect.org'); + } + elsif (defined($supplementalDetails->{'resourceID'})) + { + $w->dataElement('episode-num', "resourceID/$supplementalDetails->{'resourceID'}", 'system' => 'schedulesdirect.org'); + } + if ($newShowing) + { + $w->dataElement('episode-num', "newEpisode/true", 'system' => 'schedulesdirect.org'); + } + if (defined($programDetails->{'originalAirDate'})) + { + my $originalAirDate = $programDetails->{'originalAirDate'}; + my $d = substr($originalAirDate, 0, 4) . substr($originalAirDate, 5, 2) . substr($originalAirDate, 8, 2) . ' +0000'; + $w->dataElement('episode-num', "originalAirDate/$d", 'system' => 'schedulesdirect.org'); + } + if (defined($programDetails->{'eventDetails'}) && (ref($programDetails->{'eventDetails'}) eq 'HASH')) + { + if (defined($programDetails->{'eventDetails'}->{'venue100'})) + { + $w->dataElement('episode-num', "eventVenue/$programDetails->{'eventDetails'}->{'venue100'}", 'system' => 'schedulesdirect.org'); + } + if (defined($programDetails->{'eventDetails'}->{'gameDate'})) + { + my $gameDate = $programDetails->{'eventDetails'}->{'gameDate'}; + my $d = substr($gameDate, 0, 4) . substr($gameDate, 5, 2) . substr($gameDate, 8, 2) . ' +0000'; + $w->dataElement('episode-num', "eventDate/$d", 'system' => 'schedulesdirect.org'); + } + if (defined($programDetails->{'eventDetails'}->{'teams'}) && (ref($programDetails->{'eventDetails'}->{'teams'}) eq 'ARRAY')) + { + foreach my $t(@{$programDetails->{'eventDetails'}->{'teams'}}) + { + if (ref($t) eq 'HASH') + { + if (defined($t->{'name'})) + { + if ((defined($t->{'isHome'})) && ($t->{'isHome'})) + { + $w->dataElement('episode-num', "eventHomeTeam/$t->{'name'}", 'system' => 'schedulesdirect.org'); + } + else + { + $w->dataElement('episode-num', "eventTeam/$t->{'name'}", 'system' => 'schedulesdirect.org'); + } + } + } + } + } + if (defined($programDetails->{'eventDetails'}->{'season'}) && (ref($programDetails->{'eventDetails'}->{'season'}) eq 'HASH')) + { + if (defined($programDetails->{'eventDetails'}->{'season'}->{'season'})) + { + $w->dataElement('episode-num', "eventSeason/$programDetails->{'eventDetails'}->{'season'}->{'season'}", 'system' => 'schedulesdirect.org'); + } + if (defined($programDetails->{'eventDetails'}->{'season'}->{'type'})) + { + $w->dataElement('episode-num', "eventSeasonType/$programDetails->{'eventDetails'}->{'season'}->{'type'}", 'system' => 'schedulesdirect.org'); + } + } + } + } + if (defined($scheduleDetails->{'videoProperties'})) { $w->startTag('video'); @@ -1908,8 +2087,6 @@ # originalAirDate since generic data is not relevant for this showing. # Date transformation occurs because XMLTV uses their standardized # dates, while Schedules Direct uses YYYY-MM-DD - my $newShowing = 0; - $newShowing = $scheduleDetails->{'new'} if (defined($scheduleDetails->{'new'})); if (!$newShowing) { if (defined($programDetails->{'originalAirDate'})) @@ -2117,10 +2294,14 @@ $programsWritten++; } - $w->endTag('tv'); -$w->end(); + $DBH->commit(); + + undef $sth; -print (STDERR " $programsWritten program schedules processed\n") if (!$quiet); + print (STDERR " $programsWritten program schedules processed\n") if (!$quiet); + +$w->endTag('tv'); +$w->end(); # # Our work here is done @@ -2196,7 +2377,7 @@ SD_login(); # Login SD_downloadLineups(); # Update our SD lineups in the DB - my $sql = "select lineup, name, transport, location, details from lineups"; + my $sql = "select lineup, name, transport, location, details from lineups order by lineup"; my $sth = $DBH->prepare_cached($sql); if (!defined($sth)) { @@ -2205,11 +2386,11 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -2219,8 +2400,16 @@ $sth->bind_col( 5, undef, SQL_VARCHAR ); my $lu = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; if (scalar(@{$lu}) == 0) { @@ -2246,12 +2435,12 @@ for my $l (@{$lu}) { my $id = $l->[0]; - my $desc = "$l->[1] $l->[2] $l->[3]"; + my $lineupDesc = lineupDesc($l->[1], $l->[2], $l->[3]); $writer->write_option ( { value => $id, - text => [ [ "$id - $desc", 'en' ] ] + text => [ [ "$id - $lineupDesc", 'en' ] ] } ); } @@ -2322,6 +2511,59 @@ ); $writer->end_selectone(); + $writer->end('station-logo-order'); + } + elsif ($stage eq 'station-logo-order') + { + $writer->start_selectone + ( + { + id => 'station-logo-order', + title => [ [ 'Station logo ordering', 'en' ], ], + description => + [ [ + 'Specify the order of station logos', + 'en' + ] ], + } + ); + $writer->write_option + ( + { + value => '', + text => [ [ 'None specified (order as received)', 'en' ] ] + } + ); + $writer->write_option + ( + { + value => 'Gracenote/dark', + text => [ [ 'Gracenote/dark ordered first (Gracenote logo for dark backgrounds)', 'en' ] ] + } + ); + $writer->write_option + ( + { + value => 'Gracenote/light', + text => [ [ 'Gracenote/light ordered first (Gracenote logo for light backgrounds)', 'en' ] ] + } + ); + $writer->write_option + ( + { + value => 'Gracenote/gray', + text => [ [ 'Gracenote/gray ordered first (Gracenote logo with grayscale for light backgrounds)', 'en' ] ] + } + ); + $writer->write_option + ( + { + value => 'Gracenote/white', + text => [ [ 'Gracenote/white ordered first (Gracenote logo with all white for dark backgrounds)', 'en' ] ] + } + ); + $writer->end_selectone(); + $writer->end('select-channels'); } else @@ -2344,7 +2586,7 @@ # sub listChannels { - my ($conf, $opt, undef) = @_; + ($conf, $opt, undef) = @_; configValidate($conf, $opt); @@ -2430,6 +2672,8 @@ else { lineupValidate($conf->{'lineup'}); + my $token = DB_settingsGet('token'); + $SD->Token($token) if (defined($token)); } my $w = XML::Writer->new( 'ENCODING' => 'UTF-8', @@ -2479,7 +2723,7 @@ # # Select our lineup channels/stations # - $sql = 'select distinct channels.station, channels.channum, channels.details, stations.details from channels as channels left join stations as stations on stations.station = channels.station where channels.lineup in ( ' . join(', ', ('?') x scalar(@{$lineups})) . ' ) and channels.selected = 1'; + $sql = 'select distinct channels.station, channels.channum, channels.details, stations.details from channels as channels left join stations as stations on stations.station = channels.station where channels.lineup in ( ' . join(', ', ('?') x scalar(@{$lineups})) . ' ) and channels.selected = 1 order by channels.station'; $sth = $DBH->prepare_cached($sql); if (!defined($sth)) @@ -2496,11 +2740,11 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -2508,10 +2752,22 @@ $sth->bind_col( 3, undef, SQL_VARCHAR ); $sth->bind_col( 4, undef, SQL_VARCHAR ); - # Process each channel in our lineup - while (my $r = $sth->fetchrow_arrayref()) + my $channels = $sth->fetchall_arrayref(); + if ($sth->err()) { - my $sID = $r->[0]; + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + + $DBH->commit(); + + undef $sth; + + # Process each channel in our lineup + foreach my $r(@{$channels}) + { + my $sID = $r->[0]; my $channum = $r->[1]; my $c = $JSON->decode($r->[2]); my $s = {}; @@ -2533,20 +2789,29 @@ $writer->dataElement('display-name', $callsign) if ($callsign ne ''); $writer->dataElement('display-name', $channum) if ($channum ne ''); - # Should probably return all stationLogo's, but some applications are - # known to take the last and not the first, so return only the first - if ((defined($s->{'stationLogo'}->[0]->{'URL'})) && + # We return all stationLogo's unless asked to not return any, + # or to return only the first. + if ((defined($s->{'stationLogo'})) && + (ref($s->{'stationLogo'}) eq 'ARRAY') && (!exists($GRABBER_FIXUPS->{'NO_STATION_LOGOS'}))) { - if (defined($s->{'stationLogo'}->[0]->{'width'}) && defined($s->{'stationLogo'}->[0]->{'height'})) + for my $sl (sort { logoPriority($b) <=> logoPriority($a) } @{$s->{'stationLogo'}}) { - $writer->emptyTag('icon', 'src' => $s->{'stationLogo'}->[0]->{'URL'}, - 'width' => $s->{'stationLogo'}->[0]->{'width'}, - 'height' => $s->{'stationLogo'}->[0]->{'height'}); - } - else - { - $writer->emptyTag('icon', 'src' => $s->{'stationLogo'}->[0]->{'URL'}); + next if (ref($sl) ne 'HASH'); + if(defined($sl->{'URL'})) + { + if (defined($sl->{'width'}) && defined($sl->{'height'})) + { + $writer->emptyTag('icon', 'src' => $sl->{'URL'}, + 'width' => $sl->{'width'}, + 'height' => $sl->{'height'}); + } + else + { + $writer->emptyTag('icon', 'src' => $sl->{'URL'}); + } + last if (exists($GRABBER_FIXUPS->{'NO_MULTIPLE_STATION_LOGOS'})); + } } } @@ -2554,6 +2819,7 @@ $channelsWritten++; } + return ($channelsWritten); } @@ -2569,8 +2835,8 @@ # sub listLineups { - my ($opt, undef) = @_; - my $conf = LoadConfig($opt->{'config-file'}); + ($opt, undef) = @_; + $conf = LoadConfig($opt->{'config-file'}); my $sql; my $sth; @@ -2594,39 +2860,47 @@ DB_clean(); } - print (STDERR "Obtaining authentication token for Schedules Direct\n") if ($download && !$quiet); - SD_login() if ($download); - - my $expiry = $SD->accountExpiry; - if (!defined($expiry)) - { - print (STDERR "Unable to obtain the account expiration date: " . $SD->ErrorString . "\n"); - exit(1); - } - my $dataLastUpdated = $SD->obtainDataLastUpdated; - if (!defined($dataLastUpdated)) + if ($download) { - print (STDERR "Unable to obtain the Schedules Direct data last updated: " . $SD->ErrorString . "\n"); - exit(1); - } - my $expiryDateTime = DateTime::Format::ISO8601->parse_datetime($expiry); - my $dataLastUpdatedDateTime = DateTime::Format::ISO8601->parse_datetime($dataLastUpdated); + print (STDERR "Obtaining authentication token for Schedules Direct\n") if (!$quiet); + SD_login(); - print (STDERR " Schedules Direct account expires on " . $expiryDateTime . "\n") if (!$quiet); - print (STDERR " Schedules Direct data last updated on " . $dataLastUpdatedDateTime . "\n") if (!$quiet); + my $expiry = $SD->accountExpiry; + if (!defined($expiry)) + { + print (STDERR "Unable to obtain the account expiration date: " . $SD->ErrorString . "\n"); + exit(1); + } + my $dataLastUpdated = $SD->obtainDataLastUpdated; + if (!defined($dataLastUpdated)) + { + print (STDERR "Unable to obtain the Schedules Direct data last updated: " . $SD->ErrorString . "\n"); + exit(1); + } + my $expiryDateTime = DateTime::Format::ISO8601->parse_datetime($expiry); + my $dataLastUpdatedDateTime = DateTime::Format::ISO8601->parse_datetime($dataLastUpdated); - # - # Optimizing lineup download is simply not worth the effort - # due to having the check if any lineup has been modified - # since the last time the data was downloaded. And since - # list-lineups is expected to be used rarely, we are going - # to skip any attempt at optimization - # + print (STDERR " Schedules Direct account expires on " . $expiryDateTime . "\n") if (!$quiet); + print (STDERR " Schedules Direct data last updated on " . $dataLastUpdatedDateTime . "\n") if (!$quiet); + + # + # Optimizing lineup download is simply not worth the effort + # due to having the check if any lineup has been modified + # since the last time the data was downloaded. And since + # list-lineups is expected to be used rarely, we are going + # to skip any attempt at optimization + # - print (STDERR "Downloading lineups from Schedules Direct\n") if ($download && !$quiet); - SD_downloadLineups() if ($download); + print (STDERR "Downloading lineups from Schedules Direct\n") if (!$quiet); + SD_downloadLineups(); + } + else + { + my $token = DB_settingsGet('token'); + $SD->Token($token) if (defined($token)); + } - $sql = 'select lineup, name, transport, location, details from lineups'; + $sql = 'select lineup, name, transport, location, details from lineups order by lineup'; $sth = $DBH->prepare_cached($sql); if (!defined($sth)) { @@ -2635,11 +2909,11 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -2649,8 +2923,16 @@ $sth->bind_col( 5, undef, SQL_VARCHAR ); my $lu = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; my $w = XML::Writer->new( 'ENCODING' => 'UTF-8', 'DATA_MODE' => 1, 'DATA_INDENT' => 1, OUTPUT => 'self' ); $w->xmlDecl('UTF-8'); @@ -2665,7 +2947,7 @@ for my $l (@{$lu}) { my $id = $l->[0]; - my $lineupDesc = "$l->[1] $l->[2] $l->[3]"; + my $lineupDesc = lineupDesc($l->[1], $l->[2], $l->[3]); $w->startTag('xmltv-lineup', 'id' => $id ); my $type = mapTransport($l->[2]); $w->dataElement('type', $type); @@ -2692,7 +2974,7 @@ # sub getLineup { - my ($conf, $opt, undef) = @_; + ($conf, $opt, undef) = @_; my $sql; my $sth; @@ -2782,13 +3064,15 @@ else { lineupValidate($conf->{'lineup'}); + my $token = DB_settingsGet('token'); + $SD->Token($token) if (defined($token)); } # # Collect our lineup(s) information. # - $sql = 'select lineup, name, transport, location, details from lineups where lineup in ( ' . join(', ', ('?') x scalar(@{$conf->{'lineup'}})) . ' )'; + $sql = 'select lineup, name, transport, location, details from lineups where lineup in ( ' . join(', ', ('?') x scalar(@{$conf->{'lineup'}})) . ' ) order by lineup'; $sth = $DBH->prepare_cached($sql); if (!defined($sth)) @@ -2805,11 +3089,11 @@ } $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -2819,8 +3103,16 @@ $sth->bind_col( 5, undef, SQL_VARCHAR ); my $lu = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } - $sth->finish(); + $DBH->commit(); + + undef $sth; my $w = XML::Writer->new( 'ENCODING' => 'UTF-8', 'DATA_MODE' => 1, 'DATA_INDENT' => 1, OUTPUT => 'self' ); $w->xmlDecl('UTF-8'); @@ -2836,7 +3128,7 @@ for my $l (@{$lu}) { my $id = $l->[0]; - my $lineupDesc = "$l->[1] $l->[2] $l->[3]"; + my $lineupDesc = lineupDesc($l->[1], $l->[2], $l->[3]); $w->startTag('xmltv-lineup', 'id' => $id ); my $type = mapTransport($l->[2]); $w->dataElement('type', $type); @@ -2846,7 +3138,7 @@ # Process each channel/station in the lineup # - $sql = 'select distinct channels.station, channels.channum, channels.details, stations.details, lineups.transport from channels as channels left join stations as stations on stations.station = channels.station left join lineups as lineups on lineups.lineup = channels.lineup where channels.lineup = ? and channels.selected = 1'; + $sql = 'select distinct channels.station, channels.channum, channels.details, stations.details, lineups.transport from channels as channels left join stations as stations on stations.station = channels.station left join lineups as lineups on lineups.lineup = channels.lineup where channels.lineup = ? and channels.selected = 1 order by channels.station'; $sth = $DBH->prepare_cached($sql); if (!defined($sth)) @@ -2858,11 +3150,11 @@ $sth->bind_param( 1, $id, SQL_VARCHAR ); $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $sth->bind_col( 1, undef, SQL_VARCHAR ); @@ -2871,7 +3163,19 @@ $sth->bind_col( 4, undef, SQL_VARCHAR ); $sth->bind_col( 5, undef, SQL_VARCHAR ); - while (my $r = $sth->fetchrow_arrayref()) + my $channels = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + + $DBH->commit(); + + undef $sth; + + foreach my $r(@{$channels}) { my $sID = $r->[0]; my $channum = $r->[1]; @@ -2895,22 +3199,31 @@ $w->dataElement('name', $name) if (defined($name) && ($name ne '')); $w->dataElement('short-name', $shortname) if (defined($shortname) && ($shortname ne '')); - # get-lineup proposal is not precise, but it appears we - # should only return one logo, so return only the first - if ((defined($s->{'stationLogo'}->[0]->{'URL'})) && + + # We return all stationLogo's unless asked to not return any, + # or to return only the first. + if ((defined($s->{'stationLogo'})) && + (ref($s->{'stationLogo'}) eq 'ARRAY') && (!exists($GRABBER_FIXUPS->{'NO_STATION_LOGOS'}))) { - if (defined($s->{'stationLogo'}->[0]->{'width'}) && defined($s->{'stationLogo'}->[0]->{'height'})) - { - $w->emptyTag('logo', 'url' => $s->{'stationLogo'}->[0]->{'URL'}, - 'height' => $s->{'stationLogo'}->[0]->{'height'}, - 'width' => $s->{'stationLogo'}->[0]->{'width'}); - } - else + for my $sl (sort { logoPriority($b) <=> logoPriority($a) } @{$s->{'stationLogo'}}) { - $w->emptyTag('logo', 'url' => $s->{'stationLogo'}->[0]->{'URL'}); + next if (ref($sl) ne 'HASH'); + if(defined($sl->{'URL'})) + { + if (defined($sl->{'width'}) && defined($sl->{'height'})) + { + $w->emptyTag('logo', 'url' => $sl->{'URL'}, + 'width' => $sl->{'width'}, + 'height' => $sl->{'height'}); + } + else + { + $w->emptyTag('logo', 'url' => $sl->{'URL'}); + } + last if (exists($GRABBER_FIXUPS->{'NO_MULTIPLE_STATION_LOGOS'})); + } } - } $w->endTag('station'); @@ -3029,6 +3342,10 @@ $w->endTag('lineup-entry'); } + $DBH->commit(); + + undef $sth; + $w->endTag('xmltv-lineup'); } @@ -3071,22 +3388,41 @@ sub SD_login { my $username = DB_settingsGet('username'); - my $passwordhash = $passwordHash || DB_settingsGet('passwordhash'); + my $passwordhash = DB_settingsGet('passwordhash'); + my $pswdhash = $passwordHash || $passwordhash; + my $token = DB_settingsGet('token') if (!defined($passwordHash)); - if ((!defined($username)) || (!defined($passwordhash))) + if (!defined($username)) { - print (STDERR "Your database is not configured to access the Schedules Direct service\n"); - print (STDERR "(the username or the password hash is not available in the settings table)\n"); - print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups and then $SCRIPT_NAME --configure\n"); + print (STDERR "Your database is not configured to access the Schedules Direct\n"); + print (STDERR "service (the username is not available in the settings table).\n"); + print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups to\n"); + print (STDERR "initialize the database\n"); exit(1); } - if (!defined($SD->obtainToken($username, undef, $passwordhash))) + if (!defined($pswdhash)) + { + print (STDERR "Your database is not configured to access the Schedules Direct\n"); + print (STDERR "service automatically without manually entering the passwordhash.\n"); + print (STDERR "Either invoke the grabber specifying the --passwordhash option,\n"); + print (STDERR "or re-run $SCRIPT_NAME --manage-lineups to initialize\n"); + print (STDERR "and update the database to store the hash in the database.\n"); + exit(1); + } + + if (!defined($token = $SD->obtainToken($username, undef, $pswdhash, $token))) { print (STDERR "Unable to authenticate to Schedules Direct: " . $SD->ErrorString() . "\n"); exit(1); } + if ((defined($token)) && (defined($passwordhash)) && (!defined($passwordHash))) + { + DB_settingsSet('token', $token); + $DBH->commit(); + } + if (!defined($SD->obtainStatus())) { print (STDERR "Unable to obtain Schedules Direct server status: " . $SD->ErrorString() . "\n"); @@ -3143,25 +3479,15 @@ my $sql; my $sth; - my $accountStatus; + my $accountLineups; my $accountLineupModifiedDateTime; my $fetchRequired = 0; - $accountStatus = $SD->obtainStatus; + $accountLineups = $SD->obtainLineups(); - if (!defined($accountStatus)) + if (!defined($accountLineups)) { - print (STDERR "Unable to obtain Schedules Direct account status: " . $SD->ErrorString . "\n"); - exit(1); - } - - $sql = 'select 1 from lineups l1 where (l1.lineup = ? and l1.modified <= ?) ' . - 'union select 1 where not exists (select 1 from lineups l2 where l2.lineup = ?)'; - - $sth = $DBH->prepare_cached($sql); - if (!defined($sth)) - { - print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + print (STDERR "Unable to obtain Schedules Direct account lineups: " . $SD->ErrorString() . "\n"); exit(1); } @@ -3169,41 +3495,55 @@ for my $lineup(@{$lineups}) { undef $accountLineupModifiedDateTime; - if (defined($accountStatus->{'lineups'})) + + for my $l (@{$accountLineups}) { - for my $l (@{$accountStatus->{'lineups'}}) + next if (ref($l) ne 'HASH'); + if (defined($l->{'lineupID'}) && ($l->{'lineupID'} eq $lineup)) { - if (defined($l->{'lineup'}) && ($l->{'lineup'} eq $lineup)) - { - $accountLineupModifiedDateTime = DateTime::Format::ISO8601->parse_datetime($l->{'modified'}); - last; - } - if (defined($l->{'ID'}) && ($l->{'ID'} eq $lineup)) - { - $accountLineupModifiedDateTime = DateTime::Format::ISO8601->parse_datetime($l->{'modified'}); - last; - } + $accountLineupModifiedDateTime = DateTime::Format::ISO8601->parse_datetime($l->{'modified'}) if (defined($l->{'modified'})); + last; } } $accountLineupModifiedDateTime = $nowDateTime->clone() if (!defined($accountLineupModifiedDateTime)); + $sql = 'select 1 from lineups l1 where (l1.lineup = ? and l1.modified <= ?) ' . + 'union select 1 where not exists (select 1 from lineups l2 where l2.lineup = ?)'; + + $sth = $DBH->prepare_cached($sql); + if (!defined($sth)) + { + print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + exit(1); + } + $sth->bind_param( 1, $lineup, SQL_VARCHAR ); $sth->bind_param( 2, DateTime::Format::SQLite->format_datetime($accountLineupModifiedDateTime), SQL_DATETIME ); $sth->bind_param( 3, $lineup, SQL_VARCHAR ); $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) + $DBH->rollback(); + exit(1); } $fetchRequired |= ($sth->fetchrow_array() || 0); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } $sth->finish(); + $DBH->commit(); + + undef $sth; + } return ($fetchRequired); @@ -3225,32 +3565,26 @@ my $sql; my $sth; my $param; - my $status; + my $lineups; my @accountLineups = (); # # Delete any lineups not in our account # - $status = $SD->obtainStatus(); + $lineups = $SD->obtainLineups(); - if (!defined($status)) + if (!defined($lineups)) { - print (STDERR "Unable to obtain Schedules Direct account status: " . $SD->ErrorString() . "\n"); + print (STDERR "Unable to obtain Schedules Direct account lineups: " . $SD->ErrorString() . "\n"); exit(1); } - if (defined($status->{'lineups'})) + for my $lu(@{$lineups}) { - foreach my $alu(@{$status->{'lineups'}}) + next if (ref($lu) ne 'HASH'); + if (defined($lu->{'lineupID'})) { - if (defined($alu->{'lineup'})) - { - push(@accountLineups, $alu->{'lineup'}); - } - elsif (defined($alu->{'ID'})) - { - push(@accountLineups, $alu->{'ID'}); - } + push(@accountLineups, $lu->{'lineupID'}); } } @@ -3271,10 +3605,10 @@ } $sth->execute(); - if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } @@ -3308,20 +3642,10 @@ # # Obtain our lineups # - $lu = $SD->obtainLineups(); - if (!defined($lu)) - { - print (STDERR "Fatal error obtaining lineups: " . $SD->ErrorString() . "\n"); - print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups\n"); - print (STDERR "and/or $SCRIPT_NAME --configure\n"); - exit(1); - } - $lineups = $lu->{'lineups'}; + $lineups = $SD->obtainLineups(); if (!defined($lineups)) { - print (STDERR "Fatal error obtaining lineups\n"); - print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups\n"); - print (STDERR "and/or $SCRIPT_NAME --configure\n"); + print (STDERR "Unable to obtain Schedules Direct account lineups: " . $SD->ErrorString() . "\n"); exit(1); } @@ -3348,8 +3672,8 @@ for my $l (@{$lineups}) { - my $id = $l->{'lineup'}; - next if (!defined($id)); + next if ((ref($l) ne 'HASH') || (!defined($l->{'lineupID'}))); + my $id = $l->{'lineupID'}; my $name = $l->{'name'} || ''; my $transport = $l->{'transport'} || ''; my $location = $l->{'location'} || ''; @@ -3363,6 +3687,7 @@ if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } $sth1->bind_param( 1, $name, SQL_VARCHAR ); @@ -3374,6 +3699,7 @@ if ($sth1->err) { print (STDERR "Unexpected error when executing statement ($sql1): " . $sth1->errstr . "\n"); + $DBH->rollback(); exit(1); } } @@ -3432,16 +3758,24 @@ } $sth->bind_param( 1, $lineup, SQL_VARCHAR ); $sth->execute(); - $sth->bind_col( 1, \$lineupChannelsSelected, SQL_INTEGER ); - $sth->bind_col( 2, \$lineupTransport, SQL_VARCHAR ); - $sth->fetch(); if ($sth->err) { print (STDERR "Unexpected database error when executing statement ($sql): " . $sth->errstr . "\n"); $DBH->rollback(); exit(1); } + $sth->bind_col( 1, \$lineupChannelsSelected, SQL_INTEGER ); + $sth->bind_col( 2, \$lineupTransport, SQL_VARCHAR ); + $sth->fetchrow_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } $sth->finish(); + $DBH->commit(); + undef $sth; $sql = "create temp table if not exists channels_backup as select * from channels where 1<>1"; $sth = $DBH->prepare_cached($sql); @@ -3555,6 +3889,7 @@ if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } } @@ -3605,10 +3940,10 @@ $sth->bind_param( 2, $details, SQL_VARCHAR ); $sth->execute(); - if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } } @@ -3627,6 +3962,7 @@ if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } @@ -3654,26 +3990,29 @@ my $fatal = 0; - my $sql = 'select lineup, name, transport, location, details from lineups where lineup = ?'; - - my $sth = $DBH->prepare_cached($sql); - if (!defined($sth)) - { - print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); - exit(1); - } + my $sql; + my $sth; foreach my $lineup (@{$lineups}) { + $sql = 'select lineup, name, transport, location, details from lineups where lineup = ?'; + + $sth = $DBH->prepare_cached($sql); + if (!defined($sth)) + { + print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + exit(1); + } + $sth->bind_param( 1, $lineup, SQL_VARCHAR ); $sth->execute(); - if ($sth->err()) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); - exit(1) - } + $DBH->rollback(); + exit(1); + } $sth->bind_col( 1, undef, SQL_VARCHAR ); $sth->bind_col( 2, undef, SQL_VARCHAR ); @@ -3682,9 +4021,19 @@ $sth->bind_col( 5, undef, SQL_VARCHAR ); my $llu = $sth->fetchrow_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } $sth->finish(); + $DBH->commit(); + + undef $sth; + if (!defined($llu)) { print (STDERR "Lineup $lineup is no longer configured in your account at Schedules Direct.\n"); @@ -3816,6 +4165,9 @@ { my ($dbname, undef) = @_; + my $version; + my $rc; + # # Quick exit if we already have the database open # @@ -3841,12 +4193,13 @@ if ($@) { print (STDERR "Unable to create parent directory for $dbname: $@"); - exit(1) + exit(1); } } $DBH = DBI->connect("DBI:SQLite:dbname=$dbname", "", "", - { RaiseError => 0, PrintError => 0, AutoCommit => 0 }); + { RaiseError => 0, PrintError => 0, AutoCommit => 0, + sqlite_use_immediate_transaction => 0 }); if (!defined($DBH)) { @@ -3855,33 +4208,24 @@ } # - # SQLite specific optimizations (if it works, it works) + # Set extended timeout # - $DBH->{'AutoCommit'} = 1; - $DBH->do("PRAGMA page_size=4096"); - $DBH->do("PRAGMA auto_vacuum=2"); - $DBH->do("PRAGMA journal_mode=WAL"); - $DBH->{'AutoCommit'} = 0; + $DBH->sqlite_busy_timeout(30000); # - # Create settings tables if needed + # Validate DB version support by first checking + # if the database seems to be initialized. # - my $rc = $DBH->do("create table if not exists settings (" . - "tag varchar(256) not null primary key, " . - "value varchar(256))"); + $rc = $DBH->do("select value from settings where tag = 'version'"); if ((!defined($rc)) || ($rc < 0)) { - print (STDERR "Unable to create settings table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); + $version = 0; + } + else + { + $version = DB_settingsGet('version'); + $version = 0 if (!defined($version)); } - $DBH->commit(); - - # - # Validate DB version support - # - my $version = DB_settingsGet('version'); - $version = 0 if (!defined($version)); if ($version =~ /^\d+$/) { $version = 0 + $version; @@ -3892,142 +4236,161 @@ exit(1); } - if (0 == $version) ## Initial database creation - { - print (STDERR "Initializing database $dbname\n") if (!$quiet); - my $rc; - $rc = $DBH->do("create table lineups ( " . - "lineup varchar(128) not null primary key, " . - "name varchar(128) not null, " . - "location varchar(128) not null, " . - "transport varchar(64) not null, " . - "downloaded datetime not null default '1970-01-01 00:00:00', " . - "modified datetime not null default '1970-01-01 00:00:00', " . - "new_channels_selected integer not null default 1, " . - "details blob not null )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create lineups table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table programs ( " . - "program varchar(128) not null primary key, " . - "hash varchar(64) not null, " . - "details blob not null )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create programs table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table stations ( " . - "station varchar(128) not null primary key, " . - "details blob not null )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create stations table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table stations_schedules_hash ( " . - "station varchar(128) not null, " . - "day date not null, " . - "hash varchar(64) not null, " . - "details blob not null, " . - "primary key(station, day) )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create stations_schedules_hash table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create index stations_schedules_hash_index_hash on stations_schedules_hash (hash)"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create stations schedules hash index in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table channels ( " . - "lineup varchar(128) not null, " . - "station varchar(128) not null, " . - "channum varchar(128) not null default '', " . - "selected integer not null default 1, " . - "details blob not null )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create channels table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create index channels_index_lineup_station on channels (lineup, station)"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create channel index in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table schedules_hash ( " . - "station varchar(128) not null, " . - "day date not null, " . - "hash varchar(64) not null, " . - "primary key (station, day) )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create schedules_hash table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create index schedules_hash_index_hash on schedules_hash (hash)"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create schedules hash index in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create table schedules ( " . - "station varchar(128) not null, " . - "day date not null, " . - "starttime datetime not null, " . - "duration integer not null, " . - "program varchar(128) not null, " . - "program_hash varchar(64) not null, " . - "details blob not null, " . - "primary key (station, day, starttime, duration) )"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create schedules table in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create index schedules_index_station_starttime on schedules (station, starttime)"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create schedules index in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - $rc = $DBH->do("create index schedules_index_program on schedules (program)"); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to create schedules program index in database $dbname: " . $DBH->errstr . "\n"); - $DBH->rollback(); - exit(1); - } - - $version = 1; - DB_settingsSet('version', 1); - $DBH->commit(); - } - if ($version > $SCRIPT_DB_VERSION) { - print (STDERR "Database version $version is not supported (newer than grabber version $SCRIPT_DB_VERSION)\n"); + print (STDERR "Database version $version is not supported (newer than grabber supported version $SCRIPT_DB_VERSION)\n"); exit(1); } elsif ($version < $SCRIPT_DB_VERSION) { + if (0 == $version) ## Initial database creation + { + $version = 1; + print (STDERR "Initializing database $dbname\n") if (!$quiet); + + # + # SQLite specific optimizations that need to + # be applied at initial database creation. + # + $DBH->{'AutoCommit'} = 1; + $DBH->do('PRAGMA page_size=4096'); + $DBH->do('PRAGMA journal_mode=WAL'); + $DBH->do('PRAGMA auto_vacuum=2'); + $DBH->do('vacuum'); + $DBH->{'AutoCommit'} = 0; + + $rc = $DBH->do("create table if not exists settings (" . + "tag varchar(256) not null primary key, " . + "value varchar(256))"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create settings table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table lineups ( " . + "lineup varchar(128) not null primary key, " . + "name varchar(128) not null, " . + "location varchar(128) not null, " . + "transport varchar(64) not null, " . + "downloaded datetime not null default '1970-01-01 00:00:00', " . + "modified datetime not null default '1970-01-01 00:00:00', " . + "new_channels_selected integer not null default 1, " . + "details blob not null )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create lineups table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table programs ( " . + "program varchar(128) not null primary key, " . + "hash varchar(64) not null, " . + "details blob not null )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create programs table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table stations ( " . + "station varchar(128) not null primary key, " . + "details blob not null )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create stations table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table stations_schedules_hash ( " . + "station varchar(128) not null, " . + "day date not null, " . + "hash varchar(64) not null, " . + "details blob not null, " . + "primary key(station, day) )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create stations_schedules_hash table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create index stations_schedules_hash_index_hash on stations_schedules_hash (hash)"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create stations schedules hash index in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table channels ( " . + "lineup varchar(128) not null, " . + "station varchar(128) not null, " . + "channum varchar(128) not null default '', " . + "selected integer not null default 1, " . + "details blob not null )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create channels table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create index channels_index_lineup_station on channels (lineup, station)"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create channel index in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table schedules_hash ( " . + "station varchar(128) not null, " . + "day date not null, " . + "hash varchar(64) not null, " . + "primary key (station, day) )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create schedules_hash table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create index schedules_hash_index_hash on schedules_hash (hash)"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create schedules hash index in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create table schedules ( " . + "station varchar(128) not null, " . + "day date not null, " . + "starttime datetime not null, " . + "duration integer not null, " . + "program varchar(128) not null, " . + "program_hash varchar(64) not null, " . + "details blob not null, " . + "primary key (station, day, starttime, duration) )"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create schedules table in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create index schedules_index_station_starttime on schedules (station, starttime)"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create schedules index in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $rc = $DBH->do("create index schedules_index_program on schedules (program)"); + if ((!defined($rc)) || ($rc < 0)) + { + print (STDERR "Unable to create schedules program index in database $dbname: " . $DBH->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + DB_settingsSet('version', $version); + $DBH->commit(); + } + if (1 == $version) { $version = 2; @@ -4096,7 +4459,6 @@ $DBH->commit(); } - ## ## if (2 == $version ## Example upgrade (version 2 to 3) ## { ## $version = 3; @@ -4106,6 +4468,7 @@ ## $DBH->commit(); ## } } + $DBH->commit(); return; } @@ -4125,9 +4488,12 @@ my $value; - my $sql = "select value from settings where tag = ?"; + my $sql; + my $sth; + + $sql = "select value from settings where tag = ?"; - my $sth = $DBH->prepare_cached($sql); + $sth = $DBH->prepare_cached($sql); if (!defined($sth)) { print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); @@ -4140,42 +4506,97 @@ if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); exit(1); } $sth->bind_col( 1, \$value, SQL_VARCHAR ); - $sth->fetch(); + $sth->fetchrow_arrayref(); + if ($sth->err) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + + $sth->finish(); + + $DBH->commit(); + + undef $sth; + + return ($value); + } + +# +# DB_settingsSet +# +# Convenience routine to set a setting in the database +# +# Input: +# tag - the tag +# value - the value to set +# Output: +# +# +sub DB_settingsSet + { + my ($tag, $value, undef) = @_; + + my $sql; + my $sth; + + $sql = "replace into settings (tag, value) values (?, ?)"; + + $sth = $DBH->prepare_cached($sql); + if (!defined($sth)) + { + print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); + exit(1); + } + + $sth->bind_param( 1, $tag, SQL_VARCHAR ); + $sth->bind_param( 2, $value, SQL_VARCHAR ); + + $sth->execute(); if ($sth->err) { - print (STDERR "Unexpected error when fetching row ($sql): " . $sth->errstr . "\n"); - exit(1) + print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); } - $sth->finish(); + $DBH->commit(); + + undef $sth; - return ($value); + return; } # -# DB_settingsSet +# DB_settingsDelete # -# Convenience routine to set a setting in the database +# Convenience routine to delete a setting from the database # # Input: # tag - the tag -# value - the value to set # Output: # # -sub DB_settingsSet +sub DB_settingsDelete { - my ($tag, $value, undef) = @_; + my ($tag, undef) = @_; + + my $value; + + my $sql; + my $sth; - my $sql = "replace into settings (tag, value) values (?, ?)"; + $sql = "delete from settings where tag = ?"; - my $sth = $DBH->prepare_cached($sql); + $sth = $DBH->prepare_cached($sql); if (!defined($sth)) { print (STDERR "Unexpected error when preparing statement ($sql): " . $DBH->errstr . "\n"); @@ -4183,10 +4604,8 @@ } $sth->bind_param( 1, $tag, SQL_VARCHAR ); - $sth->bind_param( 2, $value, SQL_VARCHAR ); $sth->execute(); - if ($sth->err) { print (STDERR "Unexpected error when executing statement ($sql): " . $sth->errstr . "\n"); @@ -4196,6 +4615,10 @@ $sth->finish(); + $DBH->commit(); + + undef $sth; + return; } @@ -4429,14 +4852,15 @@ $DBH->commit(); + undef $sth; + # # Because the database may not have the needed configuration # for incremental vacuum, we issue the command, but do not # check the results of the execution (it works, or not) # $DBH->{'AutoCommit'} = 1; - $sql = "PRAGMA incremental_vacuum"; - $DBH->do($sql); + $DBH->do('PRAGMA incremental_vacuum'); $DBH->{'AutoCommit'} = 0; # @@ -4456,17 +4880,13 @@ # # In many (real world) runs, substantive data has been - # added/deleted/updated, so update any statistics for - # future optimizer choices + # added/deleted/updated, so inform sqlite to consider + # updating the database statistics for future query + # planner activity. # - $sql = "analyze"; - $rc = $DBH->do($sql); - if ((!defined($rc)) || ($rc < 0)) - { - print (STDERR "Unable to analyze data in database: " . $DBH->errstr . "\n"); - } - - $DBH->commit(); + $DBH->{'AutoCommit'} = 1; + $DBH->do('PRAGMA optimize'); + $DBH->{'AutoCommit'} = 0; return; } @@ -4489,6 +4909,7 @@ my $sql; my $rc; +# # We do not delete the lineups, channels, or stations in order to try to # preserve any channel (de)selection that may have occurred. By setting # the downloaded and modified dates to long ago, we will refresh those. @@ -4514,6 +4935,8 @@ # print (STDERR "Unable to delete stations in database: " . $DBH->errstr . "\n"); # exit(1); # } +# + $sql = "update lineups set downloaded = '1970-01-01 00:00:00', modified = '1970-01-01 00:00:00'"; $rc = $DBH->do($sql); if ((!defined($rc)) || ($rc < 0)) @@ -4549,6 +4972,7 @@ print (STDERR "Unable to delete programs in database: " . $DBH->errstr . "\n"); exit(1); } + $DBH->commit(); return; @@ -4573,12 +4997,16 @@ { my $username; my $passwordhash; + my $pswdhash; + my $token; if ((defined($conf->{'database'}->[0])) && (-f $conf->{'database'}->[0])) { DB_open($conf->{'database'}->[0]); $username = DB_settingsGet('username'); - $passwordhash = $passwordHash || DB_settingsGet('passwordhash'); + $passwordhash = DB_settingsGet('passwordhash'); + $pswdhash = $passwordHash || $passwordhash; + $token = DB_settingsGet('token') if (!defined($passwordHash)); } # Try obtained username/password, but allow re-entry @@ -4588,25 +5016,33 @@ if (!defined($username)) { $username = ask("Enter your username at Schedules Direct:"); - $passwordhash = undef; + $pswdhash = undef; $auth_prompted = 1; } - if (!defined($passwordhash)) + if (!defined($pswdhash)) { my $password = ask_password("Enter your password for $username at Schedules Direct:"); - $passwordhash = sha1_hex($password); + $pswdhash = sha1_hex($password); $auth_prompted = 1; } - last if (defined($SD->obtainToken($username, undef, $passwordhash))); + last if (defined($token = $SD->obtainToken($username, undef, $pswdhash, $token))); print (STDERR "Unable to authenticate to Schedules Direct: " . $SD->ErrorString() . "\n"); $username = undef; $passwordhash = undef; + $pswdhash = undef; + $token = undef; $auth_prompted = 1; } + if ((defined($DBH)) && (!$auth_prompted) && (defined($token)) && (defined($passwordhash)) && (!defined($passwordHash))) + { + DB_settingsSet('token', $token); + $DBH->commit(); + } + if (!defined($SD->obtainStatus())) { print (STDERR "Unable to obtain the service status at Schedules Direct: " . $SD->ErrorString() . "\n"); @@ -4625,17 +5061,10 @@ while ($choice ne 'Exit') { - my $lu = $SD->obtainLineups(); - if (!defined($lu)) - { - print (STDERR "Fatal error obtaining lineups: " . $SD->ErrorString() . "\n"); - print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups and/or $SCRIPT_NAME --configure\n"); - exit(1); - } - my $lineups = $lu->{'lineups'}; + my $lineups = $SD->obtainLineups(); if (!defined($lineups)) { - print (STDERR "Fatal error obtaining lineups\n"); + print (STDERR "Fatal error obtaining lineups: " . $SD->ErrorString() . "\n"); print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups and/or $SCRIPT_NAME --configure\n"); exit(1); } @@ -4646,19 +5075,17 @@ $prompt .= "======================================================================\n"; for my $l (@{$lineups}) { - my $desc = ''; + next if ((ref($l) ne 'HASH') || (!defined($l->{'lineupID'}))); + my $lineupDesc = ''; if (defined($l->{'isDeleted'}) && $l->{'isDeleted'}) { - $desc = "DELETED LINEUP"; + $lineupDesc = "DELETED LINEUP"; } else { - my $name = $l->{'name'} || 'None'; - my $transport = $l->{'transport'} || 'None'; - my $location = $l->{'location'} || 'None'; - $desc = "$name ($transport $location)"; + $lineupDesc = lineupDesc($l->{'name'}, $l->{'transport'}, $l->{'location'}); } - $prompt .= sprintf("%-20s %s\n", $l->{'lineup'}, $desc); + $prompt .= sprintf("%-20s %s\n", $l->{'lineupID'}, $lineupDesc); } $prompt .= "Specify a Schedules Direct account lineup management action"; @@ -4667,6 +5094,7 @@ [ 'Add', 'Add an additional lineup to your account' ], [ 'Delete', 'Delete an existing lineup from your account' ], [ 'Display Password Hash', 'Display your password hash'], + [ 'Delete Password Hash', 'Delete any password hash stored in the database'], [ 'Initialize Database' , 'Initialize/update the local database'], [ 'Channel Selection', 'Manage database lineup channel selection'], ); @@ -4707,6 +5135,11 @@ # Obtain the list of countries (by region) my $available = $SD->obtainAvailable('COUNTRIES'); + if ((!defined($available)) || (ref($available) ne 'HASH') || (scalar($available) == 0)) + { + $prompt .= "Regions are not available\n"; + next; + } @choices = (); foreach my $reg (sort(keys(%{$available}))) @@ -4809,12 +5242,12 @@ if (defined($postal_code)) { $postal_code =~ s/^\s+|\s+$//g; - # Check regex - if ("$postal_code" !~ m/$postal_code_regex/) - { - $pprompt .= "The specified postal code is not valid\n"; - $postal_code = ''; - } + # Check regex (removed due bad regex's in /available) + #if ("$postal_code" !~ m/$postal_code_regex/) + # { + # $pprompt .= "The specified postal code is not valid\n"; + # $postal_code = ''; + # } } } next if (!defined($postal_code)); @@ -4824,42 +5257,33 @@ $postal_code = $postal_code_example; } - my $headends = $SD->obtainHeadends($country_code, $postal_code); - if (!defined($headends)) + my $availablelineups = $SD->obtainLineupsAvailable($country_code, $postal_code); + if (!defined($availablelineups)) { - print (STDERR "Fatal error obtaining headends: " . $SD->ErrorString() . "\n"); - print (STDERR "Please re-run $SCRIPT_NAME --manage-lineups and/or $SCRIPT_NAME --configure\n"); + print (STDERR "Fatal error obtaining available lineups: " . $SD->ErrorString() . "\n"); exit(1); } - if ((ref($headends) ne 'ARRAY') || (scalar(@{$headends})) == 0) + if ((ref($availablelineups) ne 'ARRAY') || (scalar(@{$availablelineups})) == 0) { $prompt .= "Unable to add lineup, Schedules Direct has no lineups in $country_code/$postal_code\n"; } else { - my $location; - my $transport; my @choices = (); my $aprompt = ''; - for (my $i = 0; $i < scalar(@{$headends}); $i++) + for my $l (@{$availablelineups}) { - $transport = @{$headends}[$i]->{'transport'} || 'None'; - $location = @{$headends}[$i]->{'location'} || 'None'; - foreach my $lu (@{$headends}[$i]->{'lineups'}) + next if ((ref($l) ne 'HASH') || (!defined($l->{'lineupID'})) || ($l->{'lineupID'} eq '')); + my $lineup = $l->{'lineupID'}; + my $lineupDesc = lineupDesc($l->{'name'}, $l->{'transport'}, $l->{'location'}); + if (scalar(@choices) < 10) { - for my $l (@{$lu}) - { - my $lineup = $l->{'lineup'}; - if (scalar(@choices) < 10) - { - push (@choices, [ "$lineup", sprintf (" %-20s %s", $lineup, "$l->{'name'} ($transport $location)") ]); - } - else - { - push (@choices, [ "$lineup", sprintf ("%-20s %s", $lineup, "$l->{'name'} ($transport $location)") ]); - } - } + push (@choices, [ $lineup, sprintf (" %-20s %s", $lineup, $lineupDesc) ]); + } + else + { + push (@choices, [ $lineup, sprintf ("%-20s %s", $lineup, $lineupDesc) ]); } } @@ -4891,19 +5315,16 @@ my @choices = (); for my $l (@{$lineups}) { - my $desc = ''; + my $lineupDesc = ''; if (defined($l->{'isDeleted'}) && $l->{'isDeleted'}) { - $desc = "DELETED LINEUP"; + $lineupDesc = "DELETED LINEUP"; } else { - my $name = $l->{'name'} || 'None'; - my $transport = $l->{'transport'} || 'None'; - my $location = $l->{'location'} || 'None'; - $desc = "$name ($transport $location)"; + $lineupDesc = lineupDesc($l->{'name'}, $l->{'transport'}, $l->{'location'}); } - push (@choices, [ $l->{'lineup'}, sprintf("%-20s %s", $l->{'lineup'}, $desc) ]);; + push (@choices, [ $l->{'lineupID'}, sprintf("%-20s %s", $l->{'lineupID'}, $lineupDesc) ]);; } my $lineup_to_delete = askChoice("\nLineup to delete (ctrl-D to skip)", undef, @choices); @@ -4920,7 +5341,13 @@ } elsif ($choice eq 'Display Password Hash') { - $prompt .= "Your password hash is: $passwordhash\n"; + $prompt .= "Your password hash is: $pswdhash\n"; + } + elsif ($choice eq 'Delete Password Hash') + { + DB_settingsDelete('passwordhash'); + DB_settingsDelete('token'); + $prompt .= "Password hash deleted from the database\n"; } elsif ($choice eq 'Initialize Database') { @@ -4956,13 +5383,14 @@ if ($storehash) { - DB_settingsSet('passwordhash', $passwordhash); - $prompt .= "Schedules Direct username/passwordhash stored in database"; + DB_settingsSet('passwordhash', $pswdhash); + $prompt .= "Schedules Direct username/passwordhash stored in database\n"; } else { - DB_settingsSet('passwordhash', undef); - $prompt .= "Schedules Direct Username stored in database"; + DB_settingsDelete('passwordhash'); + DB_settingsDelete('token'); + $prompt .= "Schedules Direct username stored in database\n"; } $DBH->commit(); } @@ -4987,11 +5415,8 @@ @choices = (); for my $l (@{$lineups}) { - my $name = $l->{'name'} || 'None'; - my $transport = $l->{'transport'} || 'None'; - my $location = $l->{'location'} || 'None'; - my $desc = "$name ($transport $location)"; - push (@choices, [ $l->{'lineup'}, sprintf("%-20s %s", $l->{'lineup'}, $desc) ]);; + my $lineupDesc = lineupDesc($l->{'name'}, $l->{'transport'}, $l->{'location'}); + push (@choices, [ $l->{'lineupID'}, sprintf("%-20s %s", $l->{'lineupID'}, $lineupDesc) ]);; } $lineup = askChoice("\nLineup to manage channels (ctrl-D to skip)", undef, @choices); @@ -5122,13 +5547,7 @@ # # two by two, hands of blue # - my $rowid = 0; - my $selected = 1; - my $station = ''; - my $channum = ''; - my $cdetails = ''; - my $sdetails = ''; - my $sql = "select channels.rowid, channels.station, channels.channum, channels.selected, channels.details, stations.details from channels as channels left join stations as stations on stations.station = channels.station where channels.lineup = ?"; + my $sql = "select channels.rowid, channels.station, channels.channum, channels.selected, channels.details, stations.details from channels as channels left join stations as stations on stations.station = channels.station where channels.lineup = ? order by channels.station"; my $sth = $DBH->prepare_cached($sql); if (!defined($sth)) { @@ -5142,7 +5561,6 @@ print (STDERR "Unexpected error when preparing statement ($sqlupd): " . $DBH->errstr . "\n"); exit(1); } - $sth->bind_param( 1, $lineup, SQL_VARCHAR ); $sth->execute(); if ($sth->err) @@ -5151,17 +5569,33 @@ $DBH->rollback(); exit(1); } - $sth->bind_col( 1, \$rowid, SQL_INTEGER ); - $sth->bind_col( 2, \$station, SQL_VARCHAR ); - $sth->bind_col( 3, \$channum, SQL_VARCHAR ); - $sth->bind_col( 4, \$selected, SQL_INTEGER ); - $sth->bind_col( 5, \$cdetails, SQL_VARCHAR ); - $sth->bind_col( 6, \$sdetails, SQL_VARCHAR ); - while($sth->fetch()) + $sth->bind_col( 1, undef, SQL_INTEGER ); + $sth->bind_col( 2, undef, SQL_VARCHAR ); + $sth->bind_col( 3, undef, SQL_VARCHAR ); + $sth->bind_col( 4, undef, SQL_INTEGER ); + $sth->bind_col( 5, undef, SQL_VARCHAR ); + $sth->bind_col( 6, undef, SQL_VARCHAR ); + my $channelsSelect = $sth->fetchall_arrayref(); + if ($sth->err()) + { + print (STDERR "Unexpected error when executing fetch after execute of statement ($sql): " . $sth->errstr . "\n"); + $DBH->rollback(); + exit(1); + } + $DBH->commit(); + undef $sth; + foreach my $channelSelect(@{$channelsSelect}) { - my $c = $JSON->decode($cdetails); + my $rowid = $channelSelect->[0]; + my $station = $channelSelect->[1]; + my $channum = $channelSelect->[2]; + my $selected = $channelSelect->[3]; + my $cdetails = $channelSelect->[4]; + my $sdetails = $channelSelect->[5]; + my $c = {}; + $c = $JSON->decode($cdetails) if (defined($cdetails) && ($cdetails ne '')); my $s = {}; - $s = $JSON->decode($sdetails) if (defined($sdetails)); + $s = $JSON->decode($sdetails) if (defined($sdetails) && ($sdetails ne '')); my $name; $name = $s->{'name'} if (defined($s->{'name'})); my $callsign; @@ -5514,6 +5948,88 @@ } # +# logoPriority +# +# Return the station logo priority based on +# the configuration station-logo-order. +# +# Input: +# stationLogo - the station logo hash +# Output: +# priority - the logo priority +# +sub logoPriority + { + my ($stationLogo, undef) = @_; + my $source; + my $category; + + state $logoPrio = logoPriorityInit($conf); + + return 0 if (ref($stationLogo) ne 'HASH'); + + $source = $stationLogo->{'source'} || ''; + $source =~ s/^\s+|\s+$//g; + $category = $stationLogo->{'category'} || ''; + $category =~ s/^\s+|\s+$//g; + return $logoPrio->{"$source/$category"} || 0; + + # + # Internal one-time priority mapping initialization + # + sub logoPriorityInit + { + my $priority = {}; + my $pnum = 9999; + if (defined($conf->{'station-logo-order'}->[0])) + { + foreach my $o(split(',', $conf->{'station-logo-order'}->[0])) + { + $o =~ s/^\s+|\s+$//g; + next if ($o eq ''); + $priority->{$o} = $pnum--; + } + } + return $priority; + } + } + +# +# lineupDesc +# +# return a consistent description for a lineup +# +# Input: +# name - the lineup name/short descr +# transport - the lineup transport +# location - the lineup location +# Output: +# lineupDesc - standard description +# +sub lineupDesc + { + my ($name, $transport, $location, undef) = @_; + + my $lineupDesc = ''; + + $name = '' if (!defined($name)); + $transport = '' if (!defined($transport)); + $location = '' if (!defined($location)); + $name =~ s/^\s+|\s+$//g; + $transport =~ s/^\s+|\s+$//g; + $location =~ s/^\s+|\s+$//g; + $name = '[UPSTREAM BUG: Open ticket with Schedules Direct regarding missing name for this lineup]' if ($name eq ''); + $lineupDesc = $name; + $lineupDesc = $lineupDesc . ' (' if (($transport ne '') || ($location ne '')); + $lineupDesc = $lineupDesc . $transport if ($transport ne ''); + $lineupDesc = $lineupDesc . ' ' if (($transport ne '') && ($location ne '')); + $lineupDesc = $lineupDesc . $location if ($location ne ''); + $lineupDesc = $lineupDesc . ')' if (($transport ne '') || ($location ne '')); + + return($lineupDesc); + } + +# # A little info # =pod @@ -5631,6 +6147,10 @@ B<--passwordhash HASH> Provide the password hash on the command line. This is necessary if the hash is not stored in the database. +B<--scale-download N> Scale the download chunks from the default +sizes. A value of .5 would reduce the sizes of the chunks requested +by half. The resulting number is bound between 1 and the max value. + B<--list-channels> Write output giving elements for every channel available in the current configuration. @@ -5834,6 +6354,7 @@ # Username - set/return username to use # Password - set/return passwordhash to use # PasswordHash - set/return passwordhash to use +# Token - set/return SD Token to use # obtainToken - obtain and return SD token # obtainStatus - obtain and return SD status # isOnline - return true if SD systems online @@ -5843,7 +6364,7 @@ # deleteLineup - delete lineup from account # obtainLineups - return lineups in account # obtainLineupMaps - return maps for lineup -# obtainHeadends - return headends in country/postal +# obtainLineupsAvailable - return lineups available in country/postal # obtainStationsSchedules - return stations schedules # obtainStationsSchedulesHash - return stations schedules hash # obtainPrograms - return program data for programs @@ -5890,7 +6411,18 @@ $self->{'RaiseError'} = 0 unless $self->{'RaiseError'}; # Not (yet) implemented $self->{'PrintError'} = 0 unless $self->{'PrintError'}; # Not (yet) implemented $self->{'_Token'} = undef; - $self->{'_TokenAcquired'} = 0; # Refresh token every 12 hours + $self->{'_TokenAcquired'} = 0; # Refresh token every 20 hours + $self->{'_TokenValidated'} = 0; + if (defined($self->{'Token'})) + { + my ($token, $acquired) = split(' ', $self->{'Token'}, 2); + if (defined($token) && ($token =~ /^[0-9A-Fa-f]+$/) && defined($acquired) && ($acquired =~ /^-?\d+\.?\d*$/)) + { + $self->{'_Token'} = $token; + $self->{'_TokenAcquired'} = $acquired; + } + } + delete $self->{'Token'}; $self->{'_Error'} = 0; $self->{'_ErrorString'} = ''; $self->{'_Status'} = undef; @@ -5900,6 +6432,7 @@ $self->{'_LWP'} = LWP::UserAgent::Determined->new(agent => $self->{'UserAgent'}, conn_cache => LWP::ConnCache->new(total_capacity => $self->{'ConnCache'})); $self->{'_LWP'}->timing('1,2,5,10,20,20,20,20,20,20'); + $self->{'_LWP'}->requests_redirectable(['GET', 'HEAD', 'POST', 'PUT', 'DELETE']); $self->{'_LWP'}->default_header('Accept-Encoding' => scalar HTTP::Message::decodable(), 'Accept' => 'application/json', 'Content_Type' => 'application/json', @@ -5907,6 +6440,7 @@ 'Cache-Control' => 'no-cache'); bless($self, $class); + return $self; } @@ -6193,6 +6727,46 @@ } # +# set/return the (extended) SDToken +# +sub Token + { + my $self = shift; + print (STDERR "DEBUG: Entering " . (caller(0))[3] . " with args: \n" . Data::Dumper->new(\@_)->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + + my $return; + + if (@_) + { + $self->_resetSession; + my $t = shift; + if (defined($t)) + { + my ($token, $acquired) = split(' ', $t, 2); + if (defined($token) && ($token =~ /^[0-9A-Fa-f]+$/) && defined($acquired) && ($acquired =~ /^-?\d+\.?\d*$/)) + { + $self->{'_Token'} = $token; + $self->{'_TokenAcquired'} = $acquired; + $self->{'_TokenValidated'} = 0; + } + } + } + + if (defined($self->{'_Token'})) + { + $return = "$self->{'_Token'} $self->{'_TokenAcquired'}"; + } + else + { + $return = undef; + } + + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + + return $return; + } + +# # Resolve a possible relative uri to absolute URL # sub uriResolve @@ -6249,7 +6823,6 @@ $return = 0; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; - } $return = 0; $self->_setErrorString("Schedules Direct web services is not online"); @@ -6376,6 +6949,9 @@ return $return; } + # After add (success or failure) make sure we get a new Status + $self->{'_Status'} = undef; + my $request = HTTP::Request->new(PUT => "$self->{'RESTUrl'}/lineups/$lineup"); $request->header(Token => "$self->{'_Token'}"); @@ -6405,7 +6981,6 @@ $self->_CroakOrCarp; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; - } $return = 0; $self->_setErrorString("HTTP response content was not parseable: $responseContent"); @@ -6514,7 +7089,6 @@ $return = 0; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; - } $return = 0; $self->_setErrorString("Schedules Direct web services is not online"); @@ -6523,6 +7097,9 @@ return $return; } + # After delete (success or failure) make sure we get a new Status + $self->{'_Status'} = undef; + my $request = HTTP::Request->new(DELETE => "$self->{'RESTUrl'}/lineups/$lineup"); $request->header(Token => "$self->{'_Token'}"); @@ -6653,13 +7230,7 @@ my $r = eval { $self->{'_JSON'}->decode($responseContent) }; if (defined($r) && defined($r->{'code'}) && ($r->{'code'} == 4102)) { - $return = {}; - $return->{'datetime'} = $r->{'datetime'} || '1970-01-01T00:00:00Z'; - $return->{'serverID'} = $r->{'serverID'} || 'internal'; - $return->{'response'} = $r->{'response'} || 'NO_LINEUPS'; - $return->{'message'} = $r->{'message'} || 'No lineups have been added to this account'; - $return->{'code'} = 4102; - $return->{'lineups'} = []; + $return = []; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; } @@ -6714,7 +7285,56 @@ return $return; } - $return = $r; + if (ref($r) ne 'HASH') + { + $return = undef; + $self->_setErrorString("HTTP response content was not a hash: $responseContent"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + if (!defined($r->{'lineups'})) + { + $return = undef; + $self->_setErrorString("HTTP response content was not a hash containing a lineup entity: $responseContent"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + if (ref($r->{'lineups'}) ne 'ARRAY') + { + $return = undef; + $self->_setErrorString("HTTP response content was not a hash containing the lineup array: $responseContent"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + $return = $r->{'lineups'}; + + for my $e(@{$return}) + { + next if (!defined($e->{'lineup'})); + $e->{'lineupID'} = delete $e->{'lineup'}; + } + + if ((ref($self->{'_Status'}) eq 'HASH') && (defined($self->{'_Status'}->{'lineups'})) && (ref($self->{'_Status'}->{'lineups'}) eq 'ARRAY')) + { + for my $e(@{$return}) + { + next if (!defined($e->{'lineupID'})); + for my $se(@{$self->{'_Status'}->{'lineups'}}) + { + if ((ref($se) eq 'HASH') && (defined($se->{'lineup'})) && ($se->{'lineup'} eq $e->{'lineupID'}) && (defined($se->{'modified'}))) + { + $e->{'modified'} = $se->{'modified'}; + last; + } + } + } + } print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; @@ -6820,9 +7440,9 @@ } # -# Return list of headends in country/postal code +# Return list of lineups available in country/postal code # -sub obtainHeadends +sub obtainLineupsAvailable { my $self = shift; @@ -6834,18 +7454,18 @@ $self->_resetError; - if (!defined($country)) + if (!defined($country) || ($country eq '')) { $return = undef; - $self->_setErrorString("Country code not provided for headend list"); + $self->_setErrorString("Country code not provided for lineup list"); $self->_CroakOrCarp; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; } - if (!defined($postalcode)) + if (!defined($postalcode) || ($postalcode eq '')) { $return = undef; - $self->_setErrorString("Postal code code not provided for headend list"); + $self->_setErrorString("Postal code code not provided for lineup list"); $self->_CroakOrCarp; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; @@ -6899,12 +7519,12 @@ if ($code == 2102) { $return = []; - $self->_setErrorString("No headends in specified country/postalcode"); + $self->_setErrorString("No lineups in specified country/postalcode"); print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; } $return = undef; - $self->_setErrorString("Error obtaining headends ($code): $msg"); + $self->_setErrorString("Error obtaining lineups ($code): $msg"); $self->_CroakOrCarp; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; @@ -6955,7 +7575,34 @@ return $return; } - $return = $r; + if (ref($r) ne 'ARRAY') + { + $return = undef; + $self->_setErrorString("HTTP response content was not an array ($responseContent)"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + $return = []; + + for my $e(@{$r}) + { + next if ((ref($e) ne 'HASH') || (!defined($e->{'lineups'}))); + my $lineups = $e->{'lineups'}; + next if (ref($lineups) ne 'ARRAY'); + for my $lu(@{$lineups}) + { + my $el = {}; + next if ((ref($lu) ne 'HASH') || (!defined($lu->{'lineup'}))); + $el->{'lineupID'} = $lu->{'lineup'}; + $el->{'transport'} = $e->{'transport'} if (defined($e->{'transport'})); + $el->{'location'} = $e->{'location'} if (defined($e->{'location'})); + $el->{'name'} = $lu->{'name'} if (defined($lu->{'name'})); + $el->{'uri'} = $lu->{'uri'} if (defined($lu->{'uri'})); + push(@{$return}, $el); + } + } print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; @@ -7152,6 +7799,40 @@ return $return; } + if (ref($r) ne 'ARRAY') + { + $return = undef; + $self->_setErrorString("HTTP response content was malformed (not an array)"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + foreach my $e(@{$r}) + { + if (ref($e) ne 'HASH') + { + $return = undef; + $self->_setErrorString("HTTP response content was malformed (not an array of hashes)"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + my $hash = $e->{'metadata'}->{'md5'}; + my $startDate = $e->{'metadata'}->{'startDate'}; + my $modified = $e->{'metadata'}->{'modified'}; + $e->{'MD5'} = $hash if (defined($hash)); + $e->{'date'} = $startDate if (defined($startDate)); + $e->{'modified'} = $modified if (defined($modified)); + if ((defined($hash)) && (defined($startDate)) && (defined($modified))) + { + $e->{'code'} = 0 if (!defined($e->{'code'})); + $e->{'message'} = 'OK' if (!defined($e->{'message'})); + $e->{'response'} = 'OK' if (!defined($e->{'response'})); + } + delete $e->{'metadata'}; + } + $return = $r; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); @@ -7243,7 +7924,35 @@ return $return; } - $return = $r; + if (ref($r) ne 'HASH') + { + $return = undef; + $self->_setErrorString("HTTP response content was not a hash: $responseContent"); + $self->_CroakOrCarp; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + + $return = []; + + for my $station(keys %{$r}) + { + if (ref($r->{$station}) eq 'HASH') + { + for my $date(keys %{$r->{$station}}) + { + my $e = {}; + $e->{'stationID'} = $station; + $e->{'date'} = $date; + $e->{'code'} = $r->{$station}->{$date}->{'code'} if (defined($r->{$station}->{$date}->{'code'})); + $e->{'MD5'} = $r->{$station}->{$date}->{'md5'} if (defined($r->{$station}->{$date}->{'md5'})); + $e->{'message'} = $r->{$station}->{$date}->{'message'} if (defined($r->{$station}->{$date}->{'message'})); + $e->{'lastModified'} = $r->{$station}->{$date}->{'lastModified'} if (defined($r->{$station}->{$date}->{'lastModified'})); + $e->{'response'} = "OK" if ($r->{$station}->{$date}->{'code'} eq 0); + push(@{$return}, $e); + } + } + } print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); return $return; @@ -7587,7 +8296,7 @@ my $return; - my ($username, $password, $passwordHash, undef) = @_; + my ($username, $password, $passwordHash, $existingtoken, undef) = @_; $self->_resetError; @@ -7596,6 +8305,53 @@ $self->Username($username) if defined($username); $self->Password($password) if defined($password); $self->PasswordHash($passwordHash) if defined($passwordHash); + $self->Token($existingtoken) if defined($existingtoken); + + # Reuse existing token if available, acquired < 20 hours ago, and validated or we can validate + if (defined($self->{'_Token'}) && ($self->{'_TokenAcquired'} > ($now - 72000))) + { + if ($self->{'_TokenValidated'}) + { + print (STDERR "DEBUG: (re)using current token\n") if ($self->{'Debug'}); + $return = $self->Token; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + else + { + # Try a status request with the token + + my $request = HTTP::Request->new(GET => "$self->{'RESTUrl'}/status"); + + $request->header('Token' => "$self->{'_Token'}"); + + print (STDERR "DEBUG: HTTP request:\n" . Data::Dumper->new([$request])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + + my $response = $self->{'_LWP'}->request($request); + + print (STDERR "DEBUG: HTTP response:\n" . Data::Dumper->new([$response])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + + my $responseCode = $response->code(); + my $responseContent = $response->decoded_content(); + + print (STDERR "DEBUG: HTTP decoded response content:\n" . Data::Dumper->new([$responseContent])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + + if (($responseCode == 200) && (defined($responseContent)) && ($responseContent ne '')) + { + my $r = eval { $self->{'_JSON'}->decode($responseContent) }; + if ((defined($r)) && (ref($r) eq 'HASH') && (defined($r->{'code'})) && ($r->{'code'} == 0)) + { + $self->{'_TokenValidated'} = 1; + print (STDERR "DEBUG: (re)using validated token\n") if ($self->{'Debug'}); + $return = $self->Token; + print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); + return $return; + } + } + } + } + + $self->{'_Token'} = undef; if (!defined($self->{'Username'})) { @@ -7614,15 +8370,6 @@ return $return; } - # Reuse existing token if in current session and last token update < 12 hours ago - if (defined($self->{'_Token'}) && ($self->{'_TokenAcquired'} > ($now - 43200))) - { - print (STDERR "DEBUG: (re)using current token\n") if ($self->{'Debug'}); - $return = $self->{'_Token'}; - print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); - return $return; - } - $self->_resetSession; my $request = HTTP::Request->new(POST => "$self->{'RESTUrl'}/token"); @@ -7773,8 +8520,9 @@ $self->{'_Token'} = $token; $self->{'_TokenAcquired'} = $now; + $self->{'_TokenValidated'} = 1; - $return = $self->{'_Token'}; + $return = $self->Token; print (STDERR "DEBUG: Returning from " . (caller(0))[3] . " with: \n" . Data::Dumper->new([$return])->Pad('DEBUG: ')->Useqq(1)->Dump) if ($self->{'Debug'}); diff -Nru xmltv-0.6.1/lib/exe_wrap.pl xmltv-0.6.3/lib/exe_wrap.pl --- xmltv-0.6.1/lib/exe_wrap.pl 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/lib/exe_wrap.pl 2020-09-07 15:02:53.000000000 +0000 @@ -68,7 +68,7 @@ $cmd = shift || ""; # --version (and abbreviations thereof) -my $VERSION = '0.6.1'; +my $VERSION = '0.6.3'; if (index('--version', $cmd) == 0 and length $cmd >= 3) { print "xmltv $VERSION\n"; exit; diff -Nru xmltv-0.6.1/lib/ValidateFile.pm xmltv-0.6.3/lib/ValidateFile.pm --- xmltv-0.6.1/lib/ValidateFile.pm 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/lib/ValidateFile.pm 2020-09-07 15:02:53.000000000 +0000 @@ -74,35 +74,26 @@ The file is not well-formed XML. -=item notdtd +=item notvalid The file does not follow the XMLTV DTD. -=item unknownid +=item invalidid -No channel-entry found for a channelid that is used in a programme-entry. +An xmltvid does not look like a proper id, i.e. it does not match +/^[-a-zA-Z0-9]+(\.[-a-zA-Z0-9]+)+$/. -=item duplicatechannel +=item duplicateid More than one channel-entry found for a channelid. -=item noprogrammes - -No programme entries were found in the file. - -=item channelnoprogramme - -There are no programme entries for one of the channels listed with a -channel-entry. - -=item invalidid +=item unknownid -An xmltvid does not look like a proper id, i.e. it does not match -/^[-a-zA-Z0-9]+(\.[-a-zA-Z0-9]+)+$/. +No channel-entry found for a channelid that is used in a programme-entry. -=item noid +=item noprogrammes -A programme-entry without an id was found. +No programme entries were found in the file. =item emptytitle @@ -125,11 +116,19 @@ A programme entry with an invalid episode number was found. +=item missingtimezone + +The start/stop time for a programme entry does not include a timezone. + +=item invalidtimezone + +The start/stop time for a programme entry contains an invalid timezone. + =item badiso8859 The file is encoded in iso-8859 but contains characters that have no meaning in iso-8859 (or are control characters). -If it's iso-8859-1 aka Latin 1 it might be some characters in windows-1252 encoding. +If it's iso-8859-1 (aka Latin 1) it might be some characters in windows-1252 encoding. =item badutf8 @@ -138,6 +137,10 @@ 2) Mis-encoded single characters represented with [C3][AF][C2][BF][C2][BD] bytes 3) Mis-encoded single characters in range [C2][80-9F] +=item badentity + +The file contains one or more undefined XML entities. + =back If no errors are found, an empty list is returned. @@ -266,28 +269,12 @@ { my( $timestamp ) = @_; + # $tz is optional per the XMLTV DTD my( $date, $time, $tz ) = - ($timestamp =~ /^(\d{8})(\d{4,6})(\s+([A-Z]+|[+-]\d{4})){0,1}$/ ); + ($timestamp =~ /^(\d{8})(\d{4,6})(\s+([A-Z]+|[+-]\d{4}))?$/ ); + return 0 unless defined $date; return 0 unless defined $time; - - if( not defined( $tz ) ) - { - if( not defined( $timezoneerrors{$tz} ) ) { - w( "No timezone specified", 'missingtimezone' ); - $timezoneerrors{$tz}++; - return 0; - } - } - - if( $tz =~ /[a-zA-Z]/ ) { - if( not defined( $timezoneerrors{$tz} ) ) { - w( "Invalid timezone '$tz'", 'invalidtimezone' ); - $timezoneerrors{$tz}++; - return 0; - } - } - return 1; } diff -Nru xmltv-0.6.1/lib/XMLTV.pm.in xmltv-0.6.3/lib/XMLTV.pm.in --- xmltv-0.6.1/lib/XMLTV.pm.in 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/lib/XMLTV.pm.in 2020-09-07 15:02:53.000000000 +0000 @@ -11,7 +11,7 @@ # the xmltv package as a whole. This number should be checked by the # mkdist tool. # -our $VERSION = '0.6.1'; +our $VERSION = '0.6.3'; # Work around changing behaviour of XML::Twig. On some systems (like # mine) it always returns UTF-8 data unless KeepEncoding is specified. @@ -596,7 +596,12 @@ # my $t = new_doc_callback($my_enc_cb, $my_cred_cb, $my_ch_cb, $my_p_cb); - $t->parsefile($f); + if ($f eq '-') { + $t->parse(\*STDIN); + } + else { + $t->parsefile($f); + } } }; diff -Nru xmltv-0.6.1/Makefile.PL xmltv-0.6.3/Makefile.PL --- xmltv-0.6.1/Makefile.PL 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/Makefile.PL 2020-09-07 15:02:53.000000000 +0000 @@ -57,7 +57,7 @@ ); our $VERSION; -$VERSION = '0.6.1'; +$VERSION = '0.6.3'; # Fragment of Makefile text to give the directory where files should # be installed. The extra '.' in the middle of the path is to avoid @@ -92,7 +92,7 @@ # because it is accessed by some code we add to MakeMaker. # our @docs; -@docs = qw(doc/COPYING doc/QuickStart doc/README.win32 README); +@docs = qw(doc/COPYING doc/QuickStart doc/README.win32 README.md); # Executables to be installed. my @exes @@ -277,19 +277,19 @@ 'HTTP::Cookies' => 0, }, }, - { name => 'tv_grab_ch_search', - blurb => 'Grabber for Switzerland', - exes => [ 'grab/ch_search/tv_grab_ch_search' ], - deps => [ 'grab/ch_search/tv_grab_ch_search' => [ 'grab/ch_search/tv_grab_ch_search.in' ] ], - pl_files => { 'grab/ch_search/tv_grab_ch_search.PL' => 'grab/ch_search/tv_grab_ch_search' }, - to_clean => [ 'grab/ch_search/tv_grab_ch_search' ], - grab_need_share => [ 'ch_search' ], - prereqs => { 'HTML::Entities' => 1.27, - 'HTML::TreeBuilder' => 0, - 'HTTP::Cookies' => 0, - 'URI::Escape' => 0, - 'URI::URL' => 0, }, - }, + # { name => 'tv_grab_ch_search', + # blurb => 'Grabber for Switzerland', + # exes => [ 'grab/ch_search/tv_grab_ch_search' ], + # deps => [ 'grab/ch_search/tv_grab_ch_search' => [ 'grab/ch_search/tv_grab_ch_search.in' ] ], + # pl_files => { 'grab/ch_search/tv_grab_ch_search.PL' => 'grab/ch_search/tv_grab_ch_search' }, + # to_clean => [ 'grab/ch_search/tv_grab_ch_search' ], + # grab_need_share => [ 'ch_search' ], + # prereqs => { 'HTML::Entities' => 1.27, + # 'HTML::TreeBuilder' => 0, + # 'HTTP::Cookies' => 0, + # 'URI::Escape' => 0, + # 'URI::URL' => 0, }, + # }, { name => 'tv_grab_dk_dr', blurb => 'Grabber for Denmark (dr.dk)', @@ -298,32 +298,6 @@ 'IO::Scalar' => 0, }, }, - { name => 'tv_grab_dtv_la', - blurb => 'Grabber for Latin America & Caribbean', - exes => [ 'grab/dtv_la/tv_grab_dtv_la' ], - prereqs => { 'Date::Language' => 0, - 'Date::Parse' => 0, - 'HTML::TreeBuilder' => 0, - 'HTTP::Cookies' => 0, }, - }, - - # { name => 'tv_grab_es_laguiatv', - # blurb => 'Alternative grabber for Spain', - # exes => [ 'grab/es_laguiatv/tv_grab_es_laguiatv' ], - # prereqs => { 'DateTime' => 0, - # 'HTML::Entities' => 0, - # 'HTML::TreeBuilder' => 0, - # 'HTTP::Cache::Transparent' => 0, }, - # }, - - { name => 'tv_grab_eu_dotmedia', - blurb => 'Grabber for Europe (DEPRECATED, xmltv.se / dotmedia)', - exes => [ 'grab/eu_dotmedia/tv_grab_eu_dotmedia' ], - prereqs => { 'Compress::Zlib' => 0, - 'HTTP::Cache::Transparent' => 0, - 'IO::Scalar' => 0, }, - }, - { name => 'tv_grab_eu_epgdata', blurb => '$$ Grabber for some European countries (epgdata.com)', exes => [ 'grab/eu_epgdata/tv_grab_eu_epgdata' ], @@ -375,22 +349,16 @@ }, { name => 'tv_grab_fr', - blurb => 'Grabber for France', + blurb => 'Grabber for France (TeleStar)', exes => [ 'grab/fr/tv_grab_fr' ], prereqs => { 'DateTime' => 0, 'DateTime::Duration' => 0, 'DateTime::TimeZone' => 0, 'HTML::Entities' => 1.27, - 'HTML::TreeBuilder' => 0, }, + 'HTML::TreeBuilder' => 0, + 'HTTP::Cache::Transparent' => 1.0, }, }, - # { name => 'tv_grab_fr_kazer', - # blurb => 'Grabber for France (Kazer)', - # exes => [ 'grab/fr_kazer/tv_grab_fr_kazer' ], - # prereqs => { 'DateTime' => 0, - # 'IO::Uncompress::Unzip' => 0, }, - # }, - { name => 'tv_grab_huro', blurb => 'Grabber for Slovakia (Hungary, Romania broken)', exes => [ 'grab/huro/tv_grab_huro' ], @@ -406,21 +374,15 @@ grab_need_share => [ 'huro' ], prereqs => { 'HTML::Entities' => 0, 'HTML::TreeBuilder' => 0, + 'LWP::Protocol::https' => 0, 'Time::Piece' => 0, 'Time::Seconds' => 0, }, }, - { name => 'tv_grab_il', - blurb => 'Grabber for Israel', - exes => [ 'grab/il/tv_grab_il' ], - prereqs => { 'DateTime' => 0, }, - }, - - # { name => 'tv_grab_in_toi', - # blurb => 'Grabber for India (Times of India)', - # exes => [ 'grab/in_toi/tv_grab_in_toi' ], - # prereqs => { 'Date::Calc' => 0, - # 'URI::Escape' => 0, }, + # { name => 'tv_grab_il', + # blurb => 'Grabber for Israel', + # exes => [ 'grab/il/tv_grab_il' ], + # prereqs => { 'DateTime' => 0, }, # }, { name => 'tv_grab_is', @@ -493,58 +455,36 @@ prereqs => { 'XML::LibXML' => 0, }, }, - # { name => 'tv_grab_nl', - # blurb => 'Grabber for the Netherlands', - # exes => [ 'grab/nl/tv_grab_nl' ], - # prereqs => { 'Date::Format' => 0, - # 'Date::Parse' => 0, - # 'DateTime' => 0, - # 'HTML::Entities' => 0, - # 'HTML::TreeBuilder' => 0, - # 'HTTP::Cache::Transparent' => 0, }, + # { name => 'tv_grab_pt_meo', + # blurb => 'Grabber for Portugal (MEO)', + # exes => [ 'grab/pt_meo/tv_grab_pt_meo' ], + # prereqs => { 'DateTime' => 0, + # 'XML::LibXML' => 0, }, # }, - { name => 'tv_grab_pt_meo', - blurb => 'Grabber for Portugal (MEO)', - exes => [ 'grab/pt_meo/tv_grab_pt_meo' ], - prereqs => { 'DateTime' => 0, - 'XML::LibXML' => 0, }, - }, { name => 'tv_grab_pt_vodafone', blurb => 'Grabber for Portugal (Vodafone)', exes => [ 'grab/pt_vodafone/tv_grab_pt_vodafone' ], prereqs => { 'DateTime' => 0, 'URI::Escape' => 0, - 'XML::LibXML' => 0, }, - }, - - { name => 'tv_grab_se_swedb', - blurb => 'Grabber for Sweden', - exes => [ 'grab/se_swedb/tv_grab_se_swedb' ], - pl_files => { 'grab/se_swedb/tv_grab_se_swedb.PL' - => 'grab/se_swedb/tv_grab_se_swedb' }, - to_clean => [ 'grab/se_swedb/tv_grab_se_swedb' ], - deps => [ 'grab/se_swedb/tv_grab_se_swedb' - => [ 'grab/se_swedb/tv_grab_se_swedb.in' ] ], - prereqs => { 'Compress::Zlib' => 0, - 'HTTP::Cache::Transparent' => 0, - 'IO::Scalar' => 0, - 'XML::LibXML' => 0, }, + 'XML::LibXML' => 0, + 'DateTime::Format::Strptime' => 0, + 'URI::Encode' => 0, }, }, - { name => 'tv_grab_se_tvzon', - blurb => 'Grabber for Sweden (DEPRECATED, tvzon.se)', - exes => [ 'grab/se_tvzon/tv_grab_se_tvzon' ], - pl_files => { 'grab/se_tvzon/tv_grab_se_tvzon.PL' - => 'grab/se_tvzon/tv_grab_se_tvzon' }, - to_clean => [ 'grab/se_tvzon/tv_grab_se_tvzon' ], - deps => [ 'grab/se_tvzon/tv_grab_se_tvzon' - => [ 'grab/se_swedb/tv_grab_se_swedb.in' ] ], - prereqs => { 'Compress::Zlib' => 0, - 'HTTP::Cache::Transparent' => 0, - 'IO::Scalar' => 0, - 'XML::LibXML' => 0, }, - }, + # { name => 'tv_grab_se_swedb', + # blurb => 'Grabber for Sweden', + # exes => [ 'grab/se_swedb/tv_grab_se_swedb' ], + # pl_files => { 'grab/se_swedb/tv_grab_se_swedb.PL' + # => 'grab/se_swedb/tv_grab_se_swedb' }, + # to_clean => [ 'grab/se_swedb/tv_grab_se_swedb' ], + # deps => [ 'grab/se_swedb/tv_grab_se_swedb' + # => [ 'grab/se_swedb/tv_grab_se_swedb.in' ] ], + # prereqs => { 'Compress::Zlib' => 0, + # 'HTTP::Cache::Transparent' => 0, + # 'IO::Scalar' => 0, + # 'XML::LibXML' => 0, }, + # }, { name => 'tv_grab_tr', blurb => 'Grabber for Turkey (Digiturk)', diff -Nru xmltv-0.6.1/MANIFEST xmltv-0.6.3/MANIFEST --- xmltv-0.6.1/MANIFEST 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/MANIFEST 2020-09-07 15:02:53.000000000 +0000 @@ -1,30 +1,8 @@ -.dockerignore -.gitignore -.Dockerfile.centos-6 -.Dockerfile.centos-7 -.Dockerfile.debian-buster -.Dockerfile.debian-jessie -.Dockerfile.debian-sid -.Dockerfile.debian-stretch -.Dockerfile.fedora-24 -.Dockerfile.fedora-25 -.Dockerfile.fedora-26 -.Dockerfile.fedora-27 -.Dockerfile.fedora-28 -.Dockerfile.fedora-29 -.Dockerfile.ubuntu-artful -.Dockerfile.ubuntu-bionic -.Dockerfile.ubuntu-cosmic -.Dockerfile.ubuntu-trusty -.Dockerfile.ubuntu-xenial -.Dockerfile.ubuntu-yakkety -.Dockerfile.ubuntu-zesty -.travis.yml COPYING Changes MANIFEST Makefile.PL -README +README.md README.cygwin Uninstall.pm authors.txt @@ -85,12 +63,6 @@ grab/dk_dr/tv_grab_dk_dr grab/dk_tvtid/test.conf grab/dk_tvtid/tv_grab_dk_tvtid -grab/dtv_la/test.conf -grab/dtv_la/tv_grab_dtv_la -grab/es_laguiatv/test.conf -grab/es_laguiatv/tv_grab_es_laguiatv -grab/eu_dotmedia/test.conf -grab/eu_dotmedia/tv_grab_eu_dotmedia grab/eu_epgdata/channel_ids grab/eu_epgdata/revision_log grab/eu_epgdata/tv_grab_eu_epgdata @@ -115,8 +87,6 @@ grab/fi_sv/tv_grab_fi_sv grab/fr/test.conf grab/fr/tv_grab_fr -grab/fr_kazer/test.conf -grab/fr_kazer/tv_grab_fr_kazer grab/huro/catmap.cz grab/huro/catmap.hu grab/huro/catmap.ro @@ -127,8 +97,6 @@ grab/huro/tv_grab_huro.in grab/il/test.conf grab/il/tv_grab_il -grab/in_toi/test.conf -grab/in_toi/tv_grab_in_toi grab/is/test.conf grab/is/tv_grab_is grab/is/category_map @@ -147,8 +115,6 @@ grab/na_dtv/tv_grab_na_dtv grab/na_tvmedia/test.conf grab/na_tvmedia/tv_grab_na_tvmedia -grab/nl/test.conf -grab/nl/tv_grab_nl grab/pt_meo/test.conf grab/pt_meo/tv_grab_pt_meo grab/pt_vodafone/test.conf @@ -156,8 +122,6 @@ grab/se_swedb/test.conf grab/se_swedb/tv_grab_se_swedb.PL grab/se_swedb/tv_grab_se_swedb.in -grab/se_tvzon/test.conf -grab/se_tvzon/tv_grab_se_tvzon.PL grab/test_grabbers grab/tr/test.conf grab/tr/tv_grab_tr @@ -169,6 +133,7 @@ grab/uk_tvguide/tv_grab_uk_tvguide grab/uk_tvguide/tv_grab_uk_tvguide.map.conf grab/zz_sdjson/tv_grab_zz_sdjson +grab/zz_sdjson_sqlite/fixups.txt grab/zz_sdjson_sqlite/tv_grab_zz_sdjson_sqlite lib/Ask.pm lib/Ask/Term.pm diff -Nru xmltv-0.6.1/MANIFEST.SKIP xmltv-0.6.3/MANIFEST.SKIP --- xmltv-0.6.1/MANIFEST.SKIP 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/MANIFEST.SKIP 2020-09-07 15:02:53.000000000 +0000 @@ -2,7 +2,12 @@ (^|/)do_not_dist(/|$) ^MANIFEST\.SKIP$ ^\.git/ -^attic/ +^\.github/ +^\.gitattributes +^\.gitignore +^ci/ +^\.dockerignore +^\.travis.yml ^mkdist$ ^todo/ ^#.*#$ diff -Nru xmltv-0.6.1/mkdist xmltv-0.6.3/mkdist --- xmltv-0.6.1/mkdist 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/mkdist 1970-01-01 00:00:00.000000000 +0000 @@ -1,285 +0,0 @@ -#!/usr/bin/perl -w -# -# This script is meant to be run in an empty directory, although -# it doesn't require it. -# -# This script checks out the latest head revision (by checking it out -# from scratch), checks version information in files, ignores -# files/dirs we don't want to release, tags the source files in cvs, -# generates a ChangeLog and generates the .tar.bz2 file for release. -# Then it uploads this file to upload.sourceforge.net. -# -# It DOES NOT: add the file to the XMLTV project, or email -# announcements, or update www pages, although it reminds you to do -# so :) -# -# -- jerry@matilda.com - -use strict; -use File::Temp qw(tempdir); -use Getopt::Long; -use Date::Manip; - -#$SIG{__WARN__} = sub { die $_[0] }; - -my $debug; - -my $cvs_host = 'xmltv.cvs.sourceforge.net'; -my $tmp_checkout_dir = tempdir; - -sub Usage() -{ - print "mkdist --version [options]\n"; - print "where options are:\n"; - print " --version - creating version (eg 0.1.2)\n"; - print " --sfuser - sourceforge login with cvs access\n"; - print " defaults to first xmltv login in ~/.cvspass\n"; - print " --help - this usage message\n"; - print " --debug - print debug messages as we go\n"; - print " --login - perform cvs login to refresh/verify cvs access\n"; - print " --step - step by step confirmation of commands\n"; -} - -our ($opt_help, $opt_debug, $opt_sfuser, $opt_login, $opt_version, $opt_step); - -if ( ! GetOptions('help', 'debug', 'sfuser=s', 'login', 'version=s', 'step') ) { - Usage(); - exit(1); -} - -if ( defined($opt_help) ) { - Usage(); - exit(0); -} - -$debug=1 if ( defined($opt_debug) ); - -# Mimic Perl's system() in allowing either a single argument -# or a list of parameters (which means don't use the shell). -# -sub do_system(@) -{ - if ($opt_step) { - print "@_ [yN]? "; - my $reply = ; - return if $reply !~ /^[yY]/; - } - - print "@_\n"; - my $rc=system(@_); - if ( $rc != 0 ) { - die "command failed: $rc"; - } -} - -if ( !defined($opt_version) ) { - print STDERR "missing required argument --version, use --help for details\n"; - exit(1); -} -if ( !defined($opt_sfuser) ) { - if ( open(FD, "< $ENV{HOME}/.cvspass") ) { - while () { - if ( m/^:pserver:([^\@]+)\@$cvs_host:/o ) { - $opt_sfuser=$1; - print "assuming --sfuser $1 from first xmltv entry in ~/.cvspass\n"; - last; - } - } - close(FD); - if ( !defined($opt_sfuser) ) { - print STDERR "failed to locate any appropriate entry in ~/.cvspass\n"; - } - } - else { - print STDERR "failed to locate any appropriate entry in ~/.cvspass\n"; - } - if ( !defined($opt_sfuser) ) { - print STDERR "missing required argument --sfuser, use --help for details\n"; - exit(1); - } -} - -print "Did you update the README and README.win32 files with release information?\n"; -print "hit return to continue with mkdist, or ^C to stop here."; -my $junk=<>; # read return - -print "opt_version=\"$opt_version\"\n" if ( $debug ); - -#if ( !($opt_version=~m/^\d+\.\d+$/o) && !($opt_version=~m/^\d+\.\d+\.\d+$/o) ) { -# print STDERR "invalid version id, specify something of the form \"[0-9].[0-9]\" or \"[0-9].[0-9].[0-9]\"\n";; -# exit(1); -#} - -my $cvs_tag="V$opt_version"; -$cvs_tag=~s/\./_/og; - -print "version tag to use is: $cvs_tag\n" if ( $opt_debug ); - -# always use ssh for cvs access -$ENV{CVS_RSH}="ssh"; - -if ( defined($opt_login) ) { - # use pserver to login. FIXME Does Sourceforge support - # authenticated pserver? I thought it was anonymous only. - # - $ENV{CVSROOT}=":pserver:$opt_sfuser\@$cvs_host:/cvsroot/xmltv"; - do_system("cvs login"); -} - -print "\nChecking out current head revision in $tmp_checkout_dir/xmltv ..\n"; -chdir $tmp_checkout_dir or die "could not chdir to $tmp_checkout_dir: $!"; -$ENV{CVS_RSH} = 'ssh'; -$ENV{CVSROOT} = ":ext:$opt_sfuser\@$cvs_host:/cvsroot/xmltv"; -do_system("cvs -z7 -q co -P xmltv"); - -# Filename, and a regexp to capture the version in $1. -my %check_ver = ('README' => q{^XMLTV\s+([^, ]+)}, - 'doc/README.win32' => q{^XMLTV\s+([^, ]+)}, - 'Makefile.PL' => q{^(?:(?:our|my) )?\$VERSION\s*=\s*'(\S+)';}, - 'lib/XMLTV.pm.in' => q{^(?:(?:our|my) )?\$VERSION\s*=\s*'(\S+)';}, - 'lib/exe_wrap.pl' => q{^(?:(?:our|my) )?\$VERSION\s*=\s*'(\S+)';}, - ); -foreach my $f (keys %check_ver) { - my $re = $check_ver{$f}; - $f = "$tmp_checkout_dir/xmltv/$f"; - open(FD, $f) || die "cannot open $f: $!"; - my $got; - while () { - chomp; - if (/$re/) { - $got = $1; die if not defined $got; - if ($got ne $opt_version) { - # I think it's better not to do anything clever, but - # just let the user update the file. -- epa - die "$f:$.:found version $got, not $opt_version\n"; - } - else { last } - } - } - if (not defined $got) { - die "could not find version number in $f\n"; - } - close FD or die "cannot close $f: $!"; - print "Version number in $f is correct.\n"; -} - -# Another check on README - that the date is correct. -my $readme_f = "$tmp_checkout_dir/xmltv/README"; -open(FD, $readme_f) or die "cannot open $readme_f: $!"; -my $found = 0; -while () { - if (/^-- .+ (\d{4}-\d\d-\d\d)/) { - $found = 1; - my $date = UnixDate(ParseDate($1), '%Y-%m-%d'); - die "bad date $1 in README" if not $date; - my $today = UnixDate(ParseDate('now'), '%Y-%m-%d'); - die if not $today; - die "date in README ($date) is not today ($today),\n" - . " update the signature with date and your name\n" - if $date ne $today; - } -} -die "no signature found in README" if not $found; -close FD or die "cannot close $readme_f: $!"; - -print "\nremoving old tag $cvs_tag just in case\n"; -do_system("cvs -z3 -q tag -d \"$cvs_tag\""); - -my @toremove=grep { -e "$tmp_checkout_dir/$_" - || (warn("$_ missing, but no matter\n"), 0) } - map { "xmltv/$_" } - qw(attic - leon - cgi - todo - MANIFEST.SKIP - mkdist - ChangeLog.old); - -my $cmd="tar cf save.tar "; -for my $f (@toremove) { $cmd.="$f "; } - -$cmd.="`find xmltv -name CVS -type d`"; -$cmd.=" `find xmltv -name .cvsignore -type f`"; - -print "\ntemporarily removing files/dirs not for release..\n"; -do_system($cmd); - -# take easy route :) -do_system('rm', '-rf', map { "$tmp_checkout_dir/$_" } @toremove); - -print "\ngenerating ChangeLog..\n"; -do_system("cd xmltv && cvs2cl --utc"); - -print "\ntagging release with $cvs_tag..\n"; -do_system("cd xmltv && cvs -z3 -q tag \"$cvs_tag\""); - -print "\nremoving CVS dirs..\n"; -do_system("find xmltv -name CVS -type d -prune -exec rm -rf {} \\;"); - -print "\nrenaming xmltv directory to xmltv-$opt_version..\n"; -rename("xmltv", "xmltv-$opt_version"); - -# The tarball we generate and upload. -mkdir $opt_version; -my $filename = "xmltv-$opt_version.tar.bz2"; -print "\n"; -# 'tar --bzip2' appends useless junk to the compressed data. -do_system("tar c ./xmltv-$opt_version | bzip2 >$opt_version/$filename"); -do_system("cp xmltv-$opt_version/README $opt_version"); -print "\n"; - -# FIXME do we need this if working in a separate checkout dir? -#print "\nrestoring files that won't be released..\n"; -#rename("xmltv-$opt_version", "xmltv"); -#unlink("xmltv/ChangeLog"); -#do_system("tar xpf save.tar"); -#unlink("save.tar"); - -print "\nuploading tempdir $filename to frs.sourceforge.net..\n"; -print "scp $filename $opt_sfuser,xmltv\@frs.sourceforge.net:/home/frs/project/x/xm/xmltv/xmltv/$opt_version/\n"; -do_system( "scp -r $opt_version $opt_sfuser,xmltv\@frs.sourceforge.net:/home/frs/project/x/xm/xmltv/xmltv/"); -#do_system("cd xmltv; scp README $opt_sfuser,xmltv\@frs.sourceforge.net:/home/frs/project/x/xm/xmltv/xmltv/$opt_version/"); - -print < - (login may be required) - - change folder to xmltv/$opt_version - click on files to select appropriate attributes - - Step 1 - ------ - - change status to "Hidden" (for now) - In the release page, paste the README in as 'release notes', apart - from the 'changes in this release' section of the README, which - goes in the 'changelog' box, - NOTE: check the 'Preserve my pre-formatted text. ' box - then click 'Submit/Refresh' - - - Step 2 - ------ - - choose $filename, then click - 'Add Files and/or Refresh View' button - - - Step 3 - ------ - - set the platform 'Any' and type 'Source bz2' - - hit 'update' button - - - Toggle 'Status' in Step 1 to 'Active' and hit 'Submit/Refresh' button - at the bottom of Step 1. - < how hit 'Summary' button at top of page, you should see release - show up > - -Now update the XMLTV wiki. - -Send a release announcement to xmltv-announce\@lists.sourceforge.net, -and update the Freshmeat entry . -END - ; diff -Nru xmltv-0.6.1/README xmltv-0.6.3/README --- xmltv-0.6.1/README 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,203 +0,0 @@ -XMLTV 0.6.1 - -Gather television listings, process them and organize your viewing. -XMLTV is a file format for storing TV listings, defined in xmltv.dtd. -Then there are several tools to produce and process these listings. - -Please see doc/QuickStart for documentation on what each program does, -and xmltv.dtd for documentation on the file format. - -* Major Changes in this release (0.6.1) - -* IMPORTANT * - -tv_grab_eu_dotmedia and tv_grab_se_tvzon are deprecated and will be -removed in the next release of XMLTV. Please switch to the new -tv_grab_eu_xmltvse grabber as soon as possible. - -tv_grab_eu_xmltvse: new grabber for Europe -tv_grab_pt_vodafone: new grabber for Portugal - -tv_grab_es_laguiatv: disable broken grabber -tv_grab_fr_kazer: disable broken grabber -tv_grab_in_toi: disable broken grabber -tv_grab_nl: disable broken grabber - -tv_grab_eu_epgdata: include fanart URLs in output -tv_grab_fi: add new ampparit and telsu sources -tv_grab_il: update grabber due to upstream changes -tv_grab_is: now only provides RUV channels -tv_grab_zz_sdjson_sqlite: - improvements to lineup management - add support for TheTVDB metadata -tv_augment: new rules to improve episode numbering - logging must now be enabled explicitly -tv_count/tv_merge: mandatory command line options for files -tv_imdb: migrate to new URL for archived IMDB data - -And lots of other changes (see git log for details) - -* Installation - -Note: Windows users are strongly advised to use the pre-built EXE as installing -all the prerequisites is non-trivial. For those who want to give it a go, -instructions are in doc/exe_build.html. Those instructions can be used for both -building xmltv.exe as well as a local install. - -Basic installation instructions (Linux/Unix): - -% perl Makefile.PL -% make -% make test -% make install - -To install in a custom directory, replace the first line with -something like - -% perl Makefile.PL PREFIX=/wherever/ - -The system requirements are Perl 5.8.3 or later, and a few Perl modules. -You will be asked about some optional components; if you choose not to -install them then there are fewer dependencies. - -Please note that in addition to the specific modules listed below, the -tv_grab_zz_sdjson_sqlite grabber requires Perl 5.16 to be installed. - -** Required distribtions/modules - -Required distributions/modules for XMLTV's core libraries are: - -Date::Manip 5.42a -File::Slurp -JSON (see note below) -LWP 5.65 -Term::ReadKey -XML::LibXML -XML::Parser 2.34 -XML::TreePP -XML::Twig 3.28 -XML::Writer 0.6.0 - -Required modules for grabbers/utilities are: - -Archive::Zip (tv_grab_eu_epgdata, tv_grab_uk_bleb) -CGI (tv_pick_cgi, core module until 5.20.3, part of CGI) -CGI::Carp (tv_pick_cgi, core module until 5.20.3, part of CGI) -Compress::Zlib (for some of the grabbers, core module since 5.9.3, part of IO::Compress) -Data::Dump (for tv_grab_it_dvb) -Date::Calc (tv_grab_il) -Date::Format (for some of the grabbers, part of TimeDate) -Date::Language (tv_grab_ar, tv_grab_dtv_la, part of TimeDate) -Date::Parse (tv_grab_dtv_la, part of TimeDate) -DateTime (for several of the grabbers) -DateTime::Format::ISO8601 (tv_grab_zz_sdjson_sqlite) -DateTime::Format::SQLite (tv_grab_zz_sdjson_sqlite) -DateTime::Format::Strptime (tv_grab_eu_epgdata) -DateTime::TimeZone (tv_grab_fr) -DBD::SQLite (tv_grab_zz_sdjson_sqlite) -DBI (tv_grab_zz_sdjson_sqlite) -Digest::SHA (tv_grab_zz_sdjson{,_sqlite}, core module since 5.9.3) -File::HomeDir (tv_grab_zz_sdjson_sqlite) -File::Which (tv_grab_zz_sdjson_sqlite) -HTML::Entities 1.27 (for several of the grabbers, part of HTML::Parser 3.34) -HTML::Parser 3.34 (tv_grab_it, tv_grab_it_dvb, part of HTML::Parser 3.34) -HTML::Tree (for many of the grabbers, part of HTML::Tree) -HTML::TreeBuilder (for many of the grabbers, part of HTML::Tree) -HTTP::Cache::Transparent 1.0 (for several of the grabbers) -HTTP::Cookies (for several of the grabbers) -HTTP::Request::Common (tv_grab_eu_epgdata, part of HTTP::Message) -IO::Scalar (for some of the grabbers, part of IO::Stringy) -List::MoreUtils (tv_grab_zz_sdjson_sqlite) -LWP::Protocol::https (tv_grab_zz_sdjson) -LWP::UserAgent::Determined (tv_grab_zz_sdjson_sqlite) -SOAP::Lite 0.67 (tv_grab_na_dd) -Time::Piece (tv_grab_huro, core module since 5.9.5) -Time::Seconds (tv_grab_huro, core module since 5.9.5) -Tk (tv_check) -Tk::TableMatrix (tv_check) -URI (for some of the grabbers, part of URI) -URI::Escape (for some of the grabbers, part of URI) -XML::DOM (tv_grab_is) -XML::LibXSLT (tv_grab_is) - -** Recommended distribtions/modules - -The following modules are recommended (e.g. faster JSON processing, better -character handling) but the software will works without them installed: - -File::chdir (testing grabbers) -JSON::XS (faster JSON handling, see note below) -Lingua::Preferred 0.2.4 (helps with multilingual listings) -Log::TraceMessages (useful for debugging, not needed for normal use) -PerlIO::gzip (can make tv_imdb a bit faster) -Term::ProgressBar (displays pretty progress bars) -Unicode::String (improved character handling in tv_to_latex) - -** JSON libraries - -By default, libraries and grabbers that need to handle JSON data should specify -the JSON module. This module is a wrapper for JSON::XS-compatible modules and -supports the following JSON modules: - -JSON::XS -JSON::PP -Cpanel::JSON::XS - -JSON will use JSON::XS if available, falling back to JSON::PP (a core module -since 5.14.0) if JSON::XS is not available. Cpanel::JSON::XS can be used as an -explicit alternative by setting the PERL_JSON_BACKEND environment variable -(please refer to the JSON module's documentation for details). - - -All required modules can can be installed from CPAN using the CPAN shell program: - -% 'perl -MCPAN -e shell' - -then 'install XML::Twig' and so on. - -You may find it easier to search for packaged versions of modules from your OS -provider - software sources which distribute a packaged version of XMLTV will -often provide the modules it needs too. - -* Proxy servers - -Proxy server support is provide by the LWP modules. -You can define a proxy server via the HTTP_PROXY enviornment variable. - http_proxy=http://somehost.somedomain:port - -For more information, see the the following: -http://search.cpan.org/~gaas/libwww-perl-5.803/lib/LWP/UserAgent.pm#$ua->env_proxy - -* Known issues - -If a full HTTP URL to the XMLTV.dtd is provided in the DOCTYPE declaration of -an XMLTV document, be aware that it is possible for the link to instead -redirect to a page for accepting cookies. Such cookie-acceptance pages are more -common in Europe, and can result in applications being unable to parse the -file. - -* Author and copying - -This is free software distributed under the GPL, see COPYING. There are many -who have contributed code: they are credited in individual source files and -in the authors.txt mapping file. - -* Resources - -We have a project web page and wiki at http://www.xmltv.org - -We maintain our source code using git and our Github project is available at -https://github.com/XMLTV/xmltv. Please browse and submit new issues on our Github -issue tracker at https://github.com/XMLTV/xmltv/issues. - -We run the following mailing lists: - - xmltv-announce: Subscribe — XMLTV Release Announcements (low traffic) - xmltv-users: — XMLTV users list, mostly for problem reporting and general XMLTV questions - xmltv-devel: Subscribe — XMLTV development discussion group - -Please subscribe to any/all lists at https://sourceforge.net/p/xmltv/mailman/ - -Finally, we run an IRC channel #xmltv on Freenode. Please join us! - --- Nick Morrott, knowledgejunkie@gmail.com, 2019-02-21 diff -Nru xmltv-0.6.1/README.cygwin xmltv-0.6.3/README.cygwin --- xmltv-0.6.1/README.cygwin 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/README.cygwin 2020-09-07 15:02:53.000000000 +0000 @@ -119,7 +119,7 @@ of libxml2. This package is required for some of the grabbers (tv_grab_se) At time of writing, libxml2 version 2.5.11 is known to be OK, as is 2.6.7. At time of writing, Cygwin's default version is 2.6.4, which is known -not to work For more information on working versions see README in +not to work For more information on working versions see README.md in xml_prereq_x/XML-LibXML-xxx. For this reason you must make sure that the correct version is selected here by clicking several times on the () 'Skip'/'Version' button... @@ -183,8 +183,8 @@ cd xmltv_prereq-x ls -For more info check the README: - less README +For more info check the README.md: + less README.md Note that the packages with TK cannot be installed in the current version of Cygwin, as PERL/TK is not yet part of Cygwin... This is not diff -Nru xmltv-0.6.1/README.md xmltv-0.6.3/README.md --- xmltv-0.6.1/README.md 1970-01-01 00:00:00.000000000 +0000 +++ xmltv-0.6.3/README.md 2020-09-07 15:02:53.000000000 +0000 @@ -0,0 +1,259 @@ +

    + + + +

    + +# XMLTV 0.6.3 + +## Table of Contents +- [XMLTV 0.6.2](#xmltv-063) + * [Description](#description) + * [Changes](#changes) + * [Installation (Package)](#installation-package) + + [Linux](#linux) + + [Windows](#windows) + + [MacOS](#macos) + * [Installation (Source)](#installation-source) + + [Getting Source Code](#getting-source-code) + + [Building](#building) + + [Required distributions/modules](#required-distributionsmodules) + + [Recommended distributions/modules](#recommended-distributionsmodules) + + [JSON libraries](#json-libraries) + + [CPAN](#cpan) + + [Proxy servers](#proxy-servers) + * [Known issues](#known-issues) + * [License](#license) + * [Authors](#authors) + * [Resources](#resources) + +## Project Status + +[![Build Status](https://api.travis-ci.org/XMLTV/xmltv.svg?branch=master)](https://travis-ci.org/github/XMLTV/xmltv) + +## Description + +The XMLTV project provides a suite of software to gather television listings, process listings data, and help organize your TV viewing. + +XMLTV listings use a mature XML file format for storing TV listings, which is defined and documented in the [XMLTV DTD](xmltv.dtd). + +In addition to the many "grabbers" that provide listings for large parts of the world, there are also several tools to process and filter these listings. + +Please see our [QuickStart](doc/QuickStart) documentation for details on what each program does. + +## Changes + +To see what has changed in the current XMLTV release please check the [Changes](Changes) file. + +## Installation (Package) + +### Linux + +XMLTV is packaged for most major Linux distributions and FreeBSD. It is recommended that users install XMLTV using their preferred package manager. + +#### Debian/Ubuntu + +```bash +% sudo apt install xmltv +``` + +#### Fedora/CentOS (via RPM Fusion) + +```bash +% dnf install xmltv +``` + +### Windows + +Windows users are strongly advised to use the [pre-built binary](http://alpha-exe.xmltv.org/) as installing all prerequisites is non-trivial. + +For those who want to give it a go, please read the [EXE build instructions](doc/exe_build.html). The instructions can be used for both building xmltv.exe as well as a local install. + +### MacOS + +XMLTV is packaged for MacOS in the [Fink Project](http://pdb.finkproject.org/pdb/package.php/xmltv) + +## Installation (Source) + +### Getting Source Code + +#### Tarball/Zipfile + +The source code for the current release can be downloaded as a tarball (or zipfile) from [GitHub](https://github.com/XMLTV/xmltv/releases/latest) and extracted to a preferred location. + +#### Git + +The source code for all previous, current and future releases is available in our GitHub repository: + +```bash +% git clone https://github.com/XMLTV/xmltv.git +``` + +### Building + +To build from source please ensure all required modules are available (see below). Change to the directory containing the XMLTV source: + +```bash +% perl Makefile.PL +% make +% make test +% make install +``` + +To install to a custom directory, update the first line to provide a suitable `PREFIX` location: + +``` +% perl Makefile.PL PREFIX=/opt/xmltv/ +``` + +The system requirements are Perl 5.8.3 or later, and a few Perl modules. You will be asked about some optional components; if you choose not to install them then there are fewer dependencies. + +Please note that in addition to the specific modules listed below, the +`tv_grab_zz_sdjson_sqlite` grabber requires Perl 5.16 to be installed. + +### Required distributions/modules + +Required distributions/modules for XMLTV's core libraries are: + +```perl +Date::Manip 5.42a +File::Slurp +JSON (see note below) +LWP 5.65 +Term::ReadKey +XML::LibXML +XML::Parser 2.34 +XML::TreePP +XML::Twig 3.28 +XML::Writer 0.6.0 +``` + +Required modules for grabbers/utilities are: + +``` +Archive::Zip (tv_grab_eu_epgdata, tv_grab_uk_bleb) +CGI (tv_pick_cgi, core module until 5.20.3, part of CGI) +CGI::Carp (tv_pick_cgi, core module until 5.20.3, part of CGI) +Compress::Zlib (for some of the grabbers, core module since 5.9.3, part of IO::Compress) +Data::Dump (for tv_grab_it_dvb) +Date::Format (for some of the grabbers, part of TimeDate) +Date::Language (tv_grab_ar, part of TimeDate) +DateTime (for several of the grabbers) +DateTime::Format::ISO8601 (tv_grab_zz_sdjson_sqlite) +DateTime::Format::SQLite (tv_grab_zz_sdjson_sqlite) +DateTime::Format::Strptime (tv_grab_eu_epgdata) +DateTime::TimeZone (tv_grab_fr) +DBD::SQLite (tv_grab_zz_sdjson_sqlite) +DBI (tv_grab_zz_sdjson_sqlite) +Digest::SHA (tv_grab_zz_sdjson{,_sqlite}, core module since 5.9.3) +File::HomeDir (tv_grab_zz_sdjson_sqlite) +File::Which (tv_grab_zz_sdjson_sqlite) +HTML::Entities 1.27 (for several of the grabbers, part of HTML::Parser 3.34) +HTML::Parser 3.34 (tv_grab_it, tv_grab_it_dvb, part of HTML::Parser 3.34) +HTML::Tree (for many of the grabbers, part of HTML::Tree) +HTML::TreeBuilder (for many of the grabbers, part of HTML::Tree) +HTTP::Cache::Transparent 1.0 (for several of the grabbers) +HTTP::Cookies (for several of the grabbers) +HTTP::Request::Common (tv_grab_eu_epgdata, part of HTTP::Message) +IO::Scalar (for some of the grabbers, part of IO::Stringy) +List::MoreUtils (tv_grab_zz_sdjson_sqlite) +LWP::Protocol::https (tv_grab_fi, tv_grab_huro, tv_grab_zz_sdjson) +LWP::UserAgent::Determined (tv_grab_zz_sdjson_sqlite) +SOAP::Lite 0.67 (tv_grab_na_dd) +Time::Piece (tv_grab_huro, core module since 5.9.5) +Time::Seconds (tv_grab_huro, core module since 5.9.5) +Tk (tv_check) +Tk::TableMatrix (tv_check) +URI (for some of the grabbers, part of URI) +URI::Escape (for some of the grabbers, part of URI) +XML::DOM (tv_grab_is) +XML::LibXSLT (tv_grab_is) +``` + +When building XMLTV, any missing modules that are required for the selected grabbers/utilities will be reported. + +### Recommended distributions/modules + +The following modules are recommended but XMLTV works without them installed: + +``` +File::chdir (testing grabbers) +JSON::XS (faster JSON handling, see note below) +Lingua::Preferred 0.2.4 (helps with multilingual listings) +Log::TraceMessages (useful for debugging, not needed for normal use) +PerlIO::gzip (can make tv_imdb a bit faster) +Term::ProgressBar (displays pretty progress bars) +Unicode::String (improved character handling in tv_to_latex) +``` + +### JSON libraries + +By default, libraries and grabbers that need to handle JSON data should specify the JSON module. This module is a wrapper for JSON::XS-compatible modules and supports the following JSON modules: + +``` +JSON::XS +JSON::PP +Cpanel::JSON::XS +``` + +JSON will use JSON::XS if available, falling back to JSON::PP (a core module since 5.14.0) if JSON::XS is not available. Cpanel::JSON::XS can be used as an explicit alternative by setting the PERL_JSON_BACKEND environment variable +(please refer to the JSON module's documentation for details). + +### CPAN + +All required modules can be quickly installed from CPAN using the `cpanm` utility. For example: + +```bash +% cpanm XML::Twig +``` + +Please note that you may find it easier to search for packaged versions of required modules, as sources which distribute a packaged version of XMLTV also provide packaged versions of required modules. + +### Proxy servers + +Proxy server support is provide by the LWP modules. You can define a proxy server via the HTTP_PROXY environment variable. + +```bash +% HTTP_PROXY=http://somehost.somedomain:port +``` + +For more information, see this [article](http://search.cpan.org/~gaas/libwww-perl-5.803/lib/LWP/UserAgent.pm#$ua->env_proxy) + +## Known issues + +If a full HTTP URL to the XMLTV.dtd is provided in the DOCTYPE declaration of an XMLTV document, please be aware that it is possible for the link to instead redirect to a page for accepting cookies. Such cookie-acceptance pages are more common in Europe, and can result in applications being unable to parse the file. + +## License + +XMLTV is free software, distributed under the GNU General Public License, version 2. Please see [COPYING](COPYING) for more details. + +## Authors + +There have been many contributors to XMLTV. Where possible they are credited in individual source files and in the [authors](authors.txt) mapping file. + +## Resources + +### GitHub + +Our [GitHub project](https://github.com/XMLTV/xmltv) contains all source code, issues and Pull Requests. + +### Project Wiki + +We have a project [web page and wiki](http://www.xmltv.org) + +### Mailing Lists + +We run the following mailing lists: + +- [xmltv-users](https://sourceforge.net/projects/xmltv/lists/xmltv-users): for users to ask questions and report problems with XMLTV software + +- [xmltv-devel](https://sourceforge.net/projects/xmltv/lists/xmltv-devel): for development discussion and support + +- [xmltv-announce](https://sourceforge.net/projects/xmltv/lists/xmltv-announce): announcements of new XMLTV releases + +### IRC + +Finally, we run an IRC channel #xmltv on Freenode. Please join us! + + +-- Nick Morrott, knowledgejunkie@gmail.com, 2020-08-22 diff -Nru xmltv-0.6.1/t/test_tv_imdb.t xmltv-0.6.3/t/test_tv_imdb.t --- xmltv-0.6.1/t/test_tv_imdb.t 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/t/test_tv_imdb.t 2020-09-07 15:02:53.000000000 +0000 @@ -1,11 +1,8 @@ #!/usr/bin/perl # -# Run tv_split on some input files and check the output looks -# reasonable. This is not done by diffing against expected output but -# by reading the files generated and making sure channels and dates -# seem to match. +# Run tv_imdb on various input files and check the output is as expected. # -# -- Ed Avis, ed@membled.com, 2003-10-04 +# -- Nick Morrott , 2019-02-28 use warnings; use strict; @@ -14,14 +11,14 @@ use File::Temp qw(tempdir); use File::Copy; use XMLTV::Usage <, although date-and-time -values in XMLTV files are a subset of those allowed by XML Schema -(which itself allows a subset of ISO 8601). (Note however that the -'year' element just has a four digit year, not a full date and time.) - -Our dates use the format - - CCYY-MM-DDThh:mm:ssZ - -in other words, a bigendian date. The -, T, and : characters are -required punctuation. The Z character signifies the timezone, which -is always UTC (except for the 'code-time' element). It is up to -applications to convert to and from the local timezone if wanted - any -'tz-hint' elements in the file can be used for this. - -This DTD tries to follow the general rule that metadata is stored as -attributes while content is stored as elements. The time and channel -a programme happens to be shown are attributes of 'timeslot', other -scheduling information like whether it is a repeat are attributes of -'programme'. But the content of the programme, which is the same -whenever it is shown, is stored in subelements of 'programme'. - -Within programme data, textual element content is normally free text, -and all whitespace characters (including newline) are treated simply -as space, so unless stated otherwise the text is intended to be a -single line. Every text element has an optional 'xml:lang' attribute -associated with it (as is customary, the language should be in RFC -1766 style, eg 'en' or 'en-US'). If xml:lang is not specified then -the language of the text is unknown. However where an element has -xml:lang and contains other elements, the child elements which may -have xml:lang inherit the value from their parent, unless they -override it themselves. Semantically there is no difference between -specifying xml:lang="en" separately for every bit of free text in the -document and putting it once in the top-level 'tv' element. - -Things stored as attributes, rather than as textual content, are -typically machine-readable and must come from some limited range of -values (for example a year must be four digits). - -Many elements inside 'programme' or 'channel' are repeatable, allowing -the same thing to be stated many times. For example, a programme may -have more than one title or more than one category. The convention is -that elements appearing earlier are somehow more important or more -canonical. So you might put a programme's original title first and -its translated title second; in a cast list, the more important roles -are given first; and so on. - -I have tried to use the DTD grammar to rule out meaningless -constructs. But in some cases the grammar does allow things which -don't make any sense, for example an 'episode' element which states -neither an episode number nor an episode title. Such syntactically -sound but semantically silly things should be skipped over with a -warning by applications. - -Here is a contrived example document conforming to this DTD: - - - - - - -

    - Copyright 2004, some TV network -

    -     - - - -     - - 3SAT - - - - ARD - Das Erste - - - - - - - - blah - blah - -

    - Blah Blah Blah. -

    -     - - blah - - Mr Smith - Simon Callow - - - Colin Baker - - - - - - - The case of the bombastic blackcurrent - - - - - PG - - - -     -     - - - -     - -At present versions of this DTD correspond to releases of the 'xmltv' -package, which is a set of programs to generate and manipulate files -conforming to this DTD. The DTD is maintained by Ed Avis -(ed@membled.com) but a large part of it is based on discussions on the -xmltv-devel mailing list and elsewhere. ---> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff -Nru xmltv-0.6.1/.travis.yml xmltv-0.6.3/.travis.yml --- xmltv-0.6.1/.travis.yml 2019-02-21 04:20:39.000000000 +0000 +++ xmltv-0.6.3/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,67 +0,0 @@ ---- -notifications: - email: - recipients: - - knowledgejunkie@gmail.com - - on_success: always - on_failure: always - -sudo: required - -language: bash - -env: - - release: debian_9 - distro: debian - codename: stretch - - release: debian_10 - distro: debian - codename: buster - - release: debian_sid - distro: debian - codename: sid - - release: ubuntu_1604_lts - distro: ubuntu - codename: trusty - - release: ubuntu_1804_lts - distro: ubuntu - codename: bionic - - release: ubuntu_1810 - distro: ubuntu - codename: cosmic - - release: centos_6 - distro: centos - codename: 6 - - release: centos_7 - distro: centos - codename: 7 - - release: fedora_28 - distro: fedora - codename: 28 - - release: fedora_29 - distro: fedora - codename: 29 - -branches: - only: - - master - -services: - - docker - -before_install: - - export TZ=Europe/London - - date - - env | sort - - sudo apt-get update -qq - -install: - - sudo docker build -f .Dockerfile.${distro}-${codename} -t xmltv-build-deps/${distro}-${codename} . - - sudo docker run -t --detach -v $TRAVIS_BUILD_DIR:/src --name xmltv xmltv-build-deps/${distro}-${codename} /bin/bash - - sudo docker ps -a - -script: - - docker exec -it xmltv /bin/bash -c "cd /src && perl Makefile.PL --yes" - - docker exec -it xmltv /bin/bash -c "cd /src && make" - - docker exec -it xmltv /bin/bash -c "cd /src && make test"