swish-e/prog-bin/pdf2xml.pm

package pdf2xml;
use strict;

=pod

=head1 NAME

pdf2xml - swish-e sample module to convert pdf2xml

=head1 SYNOPSIS

    use pdf2xml;
    my $xml_record_ref = pdf2xml( $pdf_file_name );

    # or by passing content in a scalar reference
    my $xml_text_ref = pdf2xml( \$pdf_content );


=head1 DESCRIPTION

Sample module for use with other swish-e 'prog' document source programs.

Pass either a file name, or a scalar reference.

The differece is when you pass a reference to a scalar
only the content is returned.  When you pass a file name
an entire record is returned ready to be fed to swish -- this
includes the headers required by swish for indexing.

The plan is to find a library that will do this to avoid forking an external
program.

=head1 REQUIREMENTS

Uses the xpdf package that includes the pdftotext conversion program.
This is available from http://www.foolabs.com/xpdf/xpdf.html.

You will also need the module File::Temp (and its dependencies)
available from CPAN if passing content to this module (instead of a file name).


=head1 AUTHOR

Bill Moseley

=cut

use Symbol;


use vars qw(
    @ISA
    @EXPORT
    $VERSION
);

# $Id: pdf2xml.pm,v 1.3 2001/05/11 14:48:47 whmoseley Exp $
$VERSION = sprintf '%d.%02d', q$Revision: 1.3 $ =~ /: (\d+)\.(\d+)/;

require Exporter;
@ISA    = qw(Exporter);
@EXPORT = qw(pdf2xml);

my @InfoTags = qw/Title Subject Author CreationDate Creator Producer ModDate Keywords/;


sub pdf2xml {
    my $file_or_content = shift;


    my $file = ref $file_or_content
    ? create_temp_file( $file_or_content )
    : $file_or_content;

    my $headers = get_pdf_headers( $file ) || '';

    my $content_ref = get_pdf_content_ref( $file );

    my $txt = <<EOF;
<all>

<headers>
$headers
</headers>
<content>
$$content_ref
</content>
</all>
EOF

    return \$txt if ref $file_or_content;

    my $mtime  = (stat $file )[9];

    my $size = length $txt;

    my $ret = <<EOF;
Content-Length: $size
Last-Mtime: $mtime
Path-Name: $file

EOF

$ret .= $txt;

    return \$ret;
    

}

sub get_pdf_headers {

    my $file = shift;
    my $sym = gensym;

    open $sym, "pdfinfo $file |" || die "$0: Failed to open $file $!";

    my %metadata;

    while (<$sym>) {
        if ( /^\s*([^:]+):\s+(.+)$/ ) {
            my ( $metaname, $value ) = ( lc( $1 ), escapeXML( $2 ) );
            $metaname =~ tr/ /_/;
            $metadata{$metaname} = $value;
        }
    }
    close $sym or die "$0: Failed close on pipe to pdfinfo for $file: $?";

    return join "\n", map { "<$_>$metadata{$_}</$_>" } sort keys %metadata;

}

sub get_pdf_content_ref {
    my $file = shift;

    my $sym = gensym;
    open $sym, "pdftotext $file - |" or die "$0: failed to run pdftotext: $!";

    local $/ = undef;
    my $content = escapeXML(<$sym>);

    close $sym or die "$0: Failed close on pipe to pdftotext for $file: $?";

    return \$content;
}


# How are URLs printed with pdftotext?
sub escapeXML {

   my $str = shift;

   for ( $str ) {
       s/</&lt;/go;
       s/>/&gt;/go;
       tr/\014/ /; # ^L
       # s/&/&amp;/go;
       # s/"/&quot;/go;
    }
   return $str;
}

# This is the portable way to do this, I suppose.
# Otherwise, just create a file in the local directory.

sub create_temp_file {
    my $scalar_ref = shift;

    require "File/Temp.pm";

    my ( $fh, $file_name ) = File::Temp::tempfile( UNLINK => 1 );

    print $fh $$scalar_ref or die $!;


    close $fh or die "Failed to close '$file_name' $!";

    return $file_name;
}
    

1	package pdf2xml;
2	use strict;
3
4	=pod
5
6	=head1 NAME
7
8	pdf2xml - swish-e sample module to convert pdf2xml
9
10	=head1 SYNOPSIS
11
12	use pdf2xml;
13	my $xml_record_ref = pdf2xml( $pdf_file_name );
14
15	# or by passing content in a scalar reference
16	my $xml_text_ref = pdf2xml( \$pdf_content );
17
18
19
20
21	=head1 DESCRIPTION
22
23	Sample module for use with other swish-e 'prog' document source programs.
24
25	Pass either a file name, or a scalar reference.
26
27	The differece is when you pass a reference to a scalar
28	only the content is returned. When you pass a file name
29	an entire record is returned ready to be fed to swish -- this
30	includes the headers required by swish for indexing.
31
32	The plan is to find a library that will do this to avoid forking an external
33	program.
34
35	=head1 REQUIREMENTS
36
37	Uses the xpdf package that includes the pdftotext conversion program.
38	This is available from http://www.foolabs.com/xpdf/xpdf.html.
39
40	You will also need the module File::Temp (and its dependencies)
41	available from CPAN if passing content to this module (instead of a file name).
42
43
44	=head1 AUTHOR
45
46	Bill Moseley
47
48	=cut
49
50	use Symbol;
51
52
53	use vars qw(
54	@ISA
55	@EXPORT
56	$VERSION
57	);
58
59	# $Id: pdf2xml.pm,v 1.3 2001/05/11 14:48:47 whmoseley Exp $
60	$VERSION = sprintf '%d.%02d', q$Revision: 1.3 $ =~ /: (\d+)\.(\d+)/;
61
62	require Exporter;
63	@ISA = qw(Exporter);
64	@EXPORT = qw(pdf2xml);
65
66	my @InfoTags = qw/Title Subject Author CreationDate Creator Producer ModDate Keywords/;
67
68
69	sub pdf2xml {
70	my $file_or_content = shift;
71
72
73	my $file = ref $file_or_content
74	? create_temp_file( $file_or_content )
75	: $file_or_content;
76
77	my $headers = get_pdf_headers( $file ) \|\| '';
78
79	my $content_ref = get_pdf_content_ref( $file );
80
81	my $txt = <<EOF;
82	<all>
83
84	<headers>
85	$headers
86	</headers>
87	<content>
88	$$content_ref
89	</content>
90	</all>
91	EOF
92
93	return \$txt if ref $file_or_content;
94
95	my $mtime = (stat $file )[9];
96
97	my $size = length $txt;
98
99	my $ret = <<EOF;
100	Content-Length: $size
101	Last-Mtime: $mtime
102	Path-Name: $file
103
104	EOF
105
106	$ret .= $txt;
107
108	return \$ret;
109
110
111	}
112
113	sub get_pdf_headers {
114
115	my $file = shift;
116	my $sym = gensym;
117
118	open $sym, "pdfinfo $file \|" \|\| die "$0: Failed to open $file $!";
119
120	my %metadata;
121
122	while (<$sym>) {
123	if ( /^\s*([^:]+):\s+(.+)$/ ) {
124	my ( $metaname, $value ) = ( lc( $1 ), escapeXML( $2 ) );
125	$metaname =~ tr/ /_/;
126	$metadata{$metaname} = $value;
127	}
128	}
129	close $sym or die "$0: Failed close on pipe to pdfinfo for $file: $?";
130
131	return join "\n", map { "<$_>$metadata{$_}</$_>" } sort keys %metadata;
132
133	}
134
135	sub get_pdf_content_ref {
136	my $file = shift;
137
138	my $sym = gensym;
139	open $sym, "pdftotext $file - \|" or die "$0: failed to run pdftotext: $!";
140
141	local $/ = undef;
142	my $content = escapeXML(<$sym>);
143
144	close $sym or die "$0: Failed close on pipe to pdftotext for $file: $?";
145
146	return \$content;
147	}
148
149
150
151	# How are URLs printed with pdftotext?
152	sub escapeXML {
153
154	my $str = shift;
155
156	for ( $str ) {
157	s/</</go;
158	s/>/>/go;
159	tr/\014/ /; # ^L
160	# s/&/&/go;
161	# s/"/"/go;
162	}
163	return $str;
164	}
165
166	# This is the portable way to do this, I suppose.
167	# Otherwise, just create a file in the local directory.
168
169	sub create_temp_file {
170	my $scalar_ref = shift;
171
172	require "File/Temp.pm";
173
174	my ( $fh, $file_name ) = File::Temp::tempfile( UNLINK => 1 );
175
176	print $fh $$scalar_ref or die $!;
177
178
179	close $fh or die "Failed to close '$file_name' $!";
180
181	return $file_name;
182	}
183
184