001/*
002 * Copyright 2008-2011 Thomas Nichols.  http://blog.thomnichols.org
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * You are receiving this code free of charge, which represents many hours of
017 * effort from other individuals and corporations.  As a responsible member
018 * of the community, you are encouraged (but not required) to donate any
019 * enhancements or improvements back to the community under a similar open
020 * source license.  Thank you. -TMN
021 */
022
023package groovyx.net.http;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.Closeable;
028import java.io.IOException;
029import java.io.InputStream;
030import java.io.OutputStream;
031import java.io.Reader;
032import java.io.StringReader;
033import java.io.StringWriter;
034import java.io.UnsupportedEncodingException;
035import java.net.HttpURLConnection;
036import java.net.MalformedURLException;
037import java.net.URISyntaxException;
038import java.net.URLConnection;
039import java.util.ArrayList;
040import java.util.HashMap;
041import java.util.List;
042import java.util.Locale;
043import java.util.Map;
044
045import oauth.signpost.OAuthConsumer;
046import oauth.signpost.basic.DefaultOAuthConsumer;
047import oauth.signpost.basic.HttpURLConnectionRequestAdapter;
048import oauth.signpost.exception.OAuthException;
049
050import org.apache.commons.logging.Log;
051import org.apache.commons.logging.LogFactory;
052import org.apache.http.Header;
053import org.apache.http.HeaderIterator;
054import org.apache.http.HttpEntity;
055import org.apache.http.HttpResponse;
056import org.apache.http.ProtocolVersion;
057import org.apache.http.StatusLine;
058import org.apache.http.message.BasicHeader;
059import org.apache.http.message.BasicHeaderIterator;
060import org.apache.http.message.BasicStatusLine;
061import org.apache.http.params.HttpParams;
062import org.codehaus.groovy.runtime.DefaultGroovyMethods;
063import org.codehaus.groovy.runtime.EncodingGroovyMethods;
064
065/**
066 * <p>This class provides a simplified API similar to {@link HTTPBuilder}, but
067 * uses {@link java.net.HttpURLConnection} for I/O so that it is compatible
068 * with Google App Engine.  Features:
069 * <ul>
070 *  <li>Parser and Encoder support</li>
071 *  <li>Easy request and response header manipulation</li>
072 *  <li>Basic authentication</li>
073 * </ul>
074 * Notably absent are status-code based response handling and the more complex
075 * authentication mechanisms.</p>
076 *
077 * TODO request encoding support (if anyone asks for it)
078 *
079 * @see <a href='http://code.google.com/appengine/docs/java/urlfetch/overview.html'>GAE URLFetch</a>
080 * @author <a href='mailto:tomstrummer+httpbuilder@gmail.com'>Tom Nichols</a>
081 * @since 0.5.0
082 */
083public class HttpURLClient {
084
085    private Map<String,String> defaultHeaders = new HashMap<String,String>();
086    private EncoderRegistry encoderRegistry = new EncoderRegistry();
087    private ParserRegistry parserRegistry = new ParserRegistry();
088    private Object contentType = ContentType.ANY;
089    private Object requestContentType = null;
090    private URIBuilder defaultURL = null;
091    private boolean followRedirects = true;
092    protected OAuthWrapper oauth;
093
094    /** Logger instance defined for use by sub-classes */
095    protected Log log =  LogFactory.getLog( getClass() );
096
097    /**
098     * Perform a request.  Parameters are:
099     * <dl>
100     *   <dt>url</dt><dd>the entire request URL</dd>
101     *   <dt>path</dt><dd>the path portion of the request URL, if a default
102     *     URL is set on this instance.</dd>
103     *   <dt>query</dt><dd>URL query parameters for this request.</dd>
104     *   <dt>timeout</dt><dd>see {@link HttpURLConnection#setReadTimeout(int)}</dd>
105     *   <dt>method</dt><dd>This defaults to GET, or POST if a <code>body</code>
106     *   parameter is also specified.</dd>
107     *   <dt>contentType</dt><dd>Explicitly specify how to parse the response.
108     *     If this value is ContentType.ANY, the response <code>Content-Type</code>
109     *     header is used to determine how to parse the response.</dd>
110     *   <dt>requestContentType</dt><dd>used in a PUT or POST request to
111     *     transform the request body and set the proper
112     *     <code>Content-Type</code> header.  This defaults to the
113     *     <code>contentType</code> if unset.</dd>
114     *   <dt>auth</dt><dd>Basic authorization; pass the value as a list in the
115     *   form [user, pass]</dd>
116     *   <dt>headers</dt><dd>additional request headers, as a map</dd>
117     *   <dt>body</dt><dd>request content body, for a PUT or POST request.
118     *     This will be encoded using the requestContentType</dd>
119     * </dl>
120     * @param args named parameters
121     * @return the parsed response
122     * @throws URISyntaxException
123     * @throws MalformedURLException
124     * @throws IOException
125     */
126    public HttpResponseDecorator request( Map<String,?> args )
127            throws URISyntaxException, MalformedURLException, IOException {
128
129        // copy so we don't modify the original collection when removing items:
130        args = new HashMap<String,Object>(args);
131
132        Object arg = args.remove( "url" );
133        if ( arg == null && this.defaultURL == null )
134            throw new IllegalStateException( "Either the 'defaultURL' property" +
135                    " must be set or a 'url' parameter must be passed to the " +
136                    "request method." );
137        URIBuilder url = arg != null ? new URIBuilder( arg.toString() ) : defaultURL.clone();
138
139        arg = null;
140        arg = args.remove( "path" );
141        if ( arg != null ) url.setPath( arg.toString() );
142        arg = null;
143        arg = args.remove( "query" );
144        if ( arg != null ) {
145            if ( ! ( arg instanceof Map<?,?> ) )
146                throw new IllegalArgumentException( "'query' must be a map" );
147            url.setQuery( (Map<?,?>)arg );
148        }
149
150        HttpURLConnection conn = (HttpURLConnection)url.toURL().openConnection();
151        conn.setInstanceFollowRedirects( this.followRedirects );
152
153        arg = null;
154        arg = args.remove( "timeout" );
155        if ( arg != null )
156            conn.setConnectTimeout( Integer.parseInt( arg.toString() ) );
157
158        arg = null;
159        arg = args.remove( "method" );
160        if ( arg != null ) conn.setRequestMethod( arg.toString() );
161
162        arg = null;
163        arg = args.remove( "contentType" );
164        Object contentType = arg != null ? arg : this.contentType;
165        if ( contentType instanceof ContentType ) conn.addRequestProperty(
166                "Accept", ((ContentType)contentType).getAcceptHeader() );
167
168        arg = null;
169        arg = args.remove( "requestContentType" );
170        String requestContentType = arg != null ? arg.toString() :
171                this.requestContentType != null ? this.requestContentType.toString() :
172                    contentType != null ? contentType.toString() : null;
173
174        // must add default headers before setting auth:
175        for ( String key : defaultHeaders.keySet() )
176            conn.addRequestProperty( key, defaultHeaders.get( key ) );
177
178        arg = null;
179        arg = args.remove( "auth" );
180        if ( arg != null ) {
181            if ( oauth != null ) log.warn( "You are trying to use both OAuth and basic authentication!" );
182            try {
183                List<?> vals = (List<?>)arg;
184                conn.addRequestProperty( "Authorization", getBasicAuthHeader(
185                        vals.get(0).toString(), vals.get(1).toString() ) );
186            } catch ( Exception ex ) {
187                throw new IllegalArgumentException(
188                        "Auth argument must be a list in the form [user,pass]" );
189            }
190        }
191
192        arg = null;
193        arg = args.remove( "headers" );
194        if ( arg != null ) {
195            if ( ! ( arg instanceof Map<?,?> ) )
196                throw new IllegalArgumentException( "'headers' must be a map" );
197            Map<?,?> headers = (Map<?,?>)arg;
198            for ( Object key : headers.keySet() ) conn.addRequestProperty(
199                    key.toString(), headers.get( key ).toString() );
200        }
201
202
203        arg = null;
204        arg = args.remove( "body" );
205        if ( arg != null ) {  // if there is a request POST or PUT body
206            conn.setDoOutput( true );
207            final HttpEntity body = (HttpEntity)encoderRegistry.getAt(
208                    requestContentType ).call( arg );
209            // TODO configurable request charset
210
211            //TODO don't override if there is a 'content-type' in the headers list
212            conn.addRequestProperty( "Content-Type", requestContentType );
213            try {
214                // OAuth Sign if necessary.
215                if ( oauth != null ) conn = oauth.sign( conn, body );
216                // send request data
217                DefaultGroovyMethods.leftShift( conn.getOutputStream(),
218                        body.getContent() );
219            }
220            finally { conn.getOutputStream().close(); }
221        }
222        // sign the request if we're using OAuth
223        else if ( oauth != null ) conn = oauth.sign(conn, null);
224
225        if ( args.size() > 0 ) {
226            String illegalArgs = "";
227            for ( String k : args.keySet() ) illegalArgs += k + ",";
228            throw new IllegalArgumentException("Unknown named parameters: " + illegalArgs);
229        }
230
231        String method = conn.getRequestMethod();
232        log.debug( method + " " + url );
233
234        HttpResponse response = new HttpURLResponseAdapter(conn);
235        if ( ContentType.ANY.equals( contentType ) ) contentType = conn.getContentType();
236
237        Object result = this.getparsedResult(method, contentType, response);
238
239        log.debug( response.getStatusLine() );
240        HttpResponseDecorator decoratedResponse = new HttpResponseDecorator( response, result );
241
242        if ( log.isTraceEnabled() ) {
243            for ( Header h : decoratedResponse.getHeaders() )
244                log.trace( " << " + h.getName() + " : " + h.getValue() );
245        }
246
247        if ( conn.getResponseCode() > 399 )
248            throw new HttpResponseException( decoratedResponse );
249
250        return decoratedResponse;
251    }
252
253    private Object getparsedResult( String method, Object contentType, HttpResponse response )
254            throws ResponseParseException {
255
256        Object parsedData = method.equals( "HEAD" ) || method.equals( "OPTIONS" ) ?
257                null : parserRegistry.getAt( contentType ).call( response );
258        try {
259            //If response is streaming, buffer it in a byte array:
260            if ( parsedData instanceof InputStream ) {
261                ByteArrayOutputStream buffer = new ByteArrayOutputStream();
262                DefaultGroovyMethods.leftShift( buffer, (InputStream)parsedData );
263                parsedData = new ByteArrayInputStream( buffer.toByteArray() );
264            }
265            else if ( parsedData instanceof Reader ) {
266                StringWriter buffer = new StringWriter();
267                DefaultGroovyMethods.leftShift( buffer, (Reader)parsedData );
268                parsedData = new StringReader( buffer.toString() );
269            }
270            else if ( parsedData instanceof Closeable )
271                log.warn( "Parsed data is streaming, but cannot be buffered: " + parsedData.getClass() );
272            return parsedData;
273        }
274        catch ( IOException ex ) {
275            throw new ResponseParseException( new HttpResponseDecorator(response,null), ex );
276        }
277    }
278
279    private String getBasicAuthHeader( String user, String pass ) throws UnsupportedEncodingException {
280      return "Basic " + EncodingGroovyMethods.encodeBase64(
281              (user + ":" + pass).getBytes("ISO-8859-1") ).toString();
282    }
283
284    /**
285     * Set basic user and password authorization to be used for every request.
286     * Pass <code>null</code> to un-set authorization for this instance.
287     * @param user
288     * @param pass
289     * @throws UnsupportedEncodingException
290     */
291    public void setBasicAuth( Object user, Object pass ) throws UnsupportedEncodingException {
292        if ( user == null ) this.defaultHeaders.remove( "Authorization" );
293        else this.defaultHeaders.put( "Authorization",
294                getBasicAuthHeader( user.toString(), pass.toString() ) );
295    }
296
297    /**
298     * Sign all outbound requests with the given OAuth keys and tokens.  It
299     * is assumed you have already generated a consumer keypair and retrieved
300     * a proper access token pair from your target service (see
301     * <a href='http://code.google.com/p/oauth-signpost/wiki/TwitterAndSignpost'>Signpost documentation</a>
302     * for more details.)  Once this has been done all requests will be signed.
303     * @param consumerKey null if you want to _stop_ signing requests.
304     * @param consumerSecret
305     * @param accessToken
306     * @param accessSecret
307     */
308    public void setOAuth( Object consumerKey, Object consumerSecret,
309            Object accessToken, Object accessSecret ) {
310        if ( consumerKey == null ) {
311            oauth = null;
312            return;
313        }
314        this.oauth = new OAuthWrapper(consumerKey, consumerSecret, accessToken, accessSecret);
315    }
316
317    /**
318     * This class basically wraps Signpost classes so they are not loaded
319     * until {@link HttpURLClient#setOAuth(Object, Object, Object, Object)}
320     * is called.  This allows Signpost to act as an optional
321     * dependency.  If you are not using Signpost, you don't need the JAR
322     * on your classpath.
323     * @since 0.5.1
324     */
325    private static class OAuthWrapper {
326        protected OAuthConsumer oauth;
327        OAuthWrapper( Object consumerKey, Object consumerSecret,
328            Object accessToken, Object accessSecret ) {
329            oauth = new DefaultOAuthConsumer( consumerKey.toString(), consumerSecret.toString() );
330            oauth.setTokenWithSecret( accessToken.toString(), accessSecret.toString() );
331        }
332
333        HttpURLConnection sign( HttpURLConnection request, final HttpEntity body ) throws IOException {
334            try {  // OAuth Sign.
335                // Note that the request body must be repeatable even though it is an input stream.
336                if ( body == null ) return (HttpURLConnection)oauth.sign( request ).unwrap();
337                else return (HttpURLConnection)oauth.sign(
338                        new HttpURLConnectionRequestAdapter(request) {
339                            /* @Override */
340                            public InputStream getMessagePayload() throws IOException {
341                                return body.getContent();
342                            }
343                        }).unwrap();
344            }
345            catch ( final OAuthException ex ) {
346//              throw new IOException( "OAuth signing error", ex ); // 1.6 only!
347                throw new IOException( "OAuth signing error: " + ex.getMessage() ) {
348                    private static final long serialVersionUID = -13848840190384656L;
349                    /* @Override */ public Throwable getCause() { return ex; }
350                };
351            }
352        }
353    }
354
355    /**
356     * Control whether this instance should automatically follow redirect
357     * responses. See {@link HttpURLConnection#setInstanceFollowRedirects(boolean)}
358     * @param follow true if the connection should automatically follow
359     * redirect responses from the server.
360     */
361    public void setFollowRedirects( boolean follow ) {
362        this.followRedirects = follow;
363    }
364
365    /**
366     * See {@link #setFollowRedirects(boolean)}
367     * @return
368     */
369    public boolean isFollowRedirects() { return this.followRedirects; }
370
371    /**
372     * The default URL for this request.  This is a {@link URIBuilder} which can
373     * be used to easily manipulate portions of the request URL.
374     * @return
375     */
376    public Object getUrl() { return this.defaultURL; }
377
378    /**
379     * Set the default request URL.
380     * @see URIBuilder#convertToURI(Object)
381     * @param url any object whose <code>toString()</code> produces a valid URI.
382     * @throws URISyntaxException
383     */
384    public void setUrl( Object url ) throws URISyntaxException {
385        this.defaultURL = new URIBuilder( URIBuilder.convertToURI( url ) );
386    }
387
388    /**
389     * This class makes a HttpURLConnection look like an HttpResponse for use
390     * by {@link ParserRegistry} and {@link HttpResponseDecorator}.
391     */
392    private final class HttpURLResponseAdapter implements HttpResponse {
393
394        HttpURLConnection conn;
395        Header[] headers;
396
397        HttpURLResponseAdapter( HttpURLConnection conn ) {
398            this.conn = conn;
399        }
400
401        public HttpEntity getEntity() {
402            return new HttpEntity() {
403
404                public void consumeContent() throws IOException {
405                    conn.getInputStream().close();
406                }
407
408                public InputStream getContent()
409                        throws IOException, IllegalStateException {
410                    if ( Status.find( conn.getResponseCode() )
411                            == Status.FAILURE ) return conn.getErrorStream();
412                    return conn.getInputStream();
413                }
414
415                public Header getContentEncoding() {
416                    return new BasicHeader( "Content-Encoding",
417                            conn.getContentEncoding() );
418                }
419
420                public long getContentLength() {
421                    return conn.getContentLength();
422                }
423
424                public Header getContentType() {
425                    return new BasicHeader( "Content-Type", conn.getContentType() );
426                }
427
428                public boolean isChunked() {
429                    String enc = conn.getHeaderField( "Transfer-Encoding" );
430                    return enc != null && enc.contains( "chunked" );
431                }
432
433                public boolean isRepeatable() {
434                    return false;
435                }
436
437                public boolean isStreaming() {
438                    return true;
439                }
440
441                public void writeTo( OutputStream out ) throws IOException {
442                    DefaultGroovyMethods.leftShift( out, conn.getInputStream() );
443                }
444
445            };
446        }
447
448        public Locale getLocale() {  //TODO test me
449            String val = conn.getHeaderField( "Locale" );
450            return val != null ? new Locale( val ) : Locale.getDefault();
451        }
452
453        public StatusLine getStatusLine() {
454            try {
455                return new BasicStatusLine( this.getProtocolVersion(),
456                    conn.getResponseCode(), conn.getResponseMessage() );
457            } catch ( IOException ex ) {
458                throw new RuntimeException( "Error reading status line", ex );
459            }
460        }
461
462        public boolean containsHeader( String key ) {
463            return conn.getHeaderField( key ) != null;
464        }
465
466        public Header[] getAllHeaders() {
467            if ( this.headers != null ) return this.headers;
468            List<Header> headers = new ArrayList<Header>();
469
470            // see http://java.sun.com/j2se/1.5.0/docs/api/java/net/HttpURLConnection.html#getHeaderFieldKey(int)
471            int i= conn.getHeaderFieldKey( 0 ) != null ? 0 : 1;
472            String key;
473            while ( ( key = conn.getHeaderFieldKey( i ) ) != null ) {
474                headers.add( new BasicHeader( key, conn.getHeaderField( i++ ) ) );
475            }
476
477            this.headers = headers.toArray( new Header[headers.size()] );
478            return this.headers;
479        }
480
481        public Header getFirstHeader( String key ) {
482            for ( Header h : getAllHeaders() )
483                if ( h.getName().equals( key ) ) return h;
484            return null;
485        }
486
487        /**
488         * Note that HttpURLConnection does not support multiple headers of
489         * the same name.
490         */
491        public Header[] getHeaders( String key ) {
492            List<Header> headers = new ArrayList<Header>();
493            for ( Header h : getAllHeaders() )
494                if ( h.getName().equals( key ) ) headers.add( h );
495            return headers.toArray( new Header[headers.size()] );
496        }
497
498        /**
499         * @see URLConnection#getHeaderField(String)
500         */
501        public Header getLastHeader( String key ) {
502            String val = conn.getHeaderField( key );
503            return val != null ? new BasicHeader( key, val ) : null;
504        }
505
506        public HttpParams getParams() { return null; }
507
508        public ProtocolVersion getProtocolVersion() {
509            /* TODO this could potentially cause problems if the server is
510               using HTTP 1.0 */
511            return new ProtocolVersion( "HTTP", 1, 1 );
512        }
513
514        public HeaderIterator headerIterator() {
515            return new BasicHeaderIterator( this.getAllHeaders(), null );
516        }
517
518        public HeaderIterator headerIterator( String key ) {
519            return new BasicHeaderIterator( this.getHeaders( key ), key );
520        }
521
522        /* Setters are part of the interface, but aren't applicable for this
523         * adapter */
524        public void setEntity( HttpEntity entity ) {}
525        public void setLocale( Locale l ) {}
526        public void setReasonPhrase( String phrase ) {}
527        public void setStatusCode( int code ) {}
528        public void setStatusLine( StatusLine line ) {}
529        public void setStatusLine( ProtocolVersion v, int code ) {}
530        public void setStatusLine( ProtocolVersion arg0,
531                int arg1, String arg2 ) {}
532        public void addHeader( Header arg0 ) {}
533        public void addHeader( String arg0, String arg1 ) {}
534        public void removeHeader( Header arg0 ) {}
535        public void removeHeaders( String arg0 ) {}
536        public void setHeader( Header arg0 ) {}
537        public void setHeader( String arg0, String arg1 ) {}
538        public void setHeaders( Header[] arg0 ) {}
539        public void setParams( HttpParams arg0 ) {}
540    }
541
542    /**
543     * Retrieve the default headers that will be sent in each request.  Note
544     * that this is a 'live' map that can be directly manipulated to add or
545     * remove the default request headers.
546     * @return
547     */
548    public Map<String,String> getHeaders() {
549        return defaultHeaders;
550    }
551
552    /**
553     * Set default headers to be sent with every request.
554     * @param headers
555     */
556    public void setHeaders( Map<?,?> headers ) {
557        this.defaultHeaders.clear();
558        for ( Object key : headers.keySet() ) {
559            Object val = headers.get( key );
560            if ( val != null ) this.defaultHeaders.put(
561                    key.toString(), val.toString() );
562        }
563    }
564
565    /**
566     * Get the encoder registry used by this instance, which can be used
567     * to directly modify the request serialization behavior.
568     * i.e. <code>client.encoders.'application/xml' = {....}</code>.
569     * @return
570     */
571    public EncoderRegistry getEncoders() {
572        return encoderRegistry;
573    }
574
575    public void setEncoders( EncoderRegistry encoderRegistry ) {
576        this.encoderRegistry = encoderRegistry;
577    }
578
579    /**
580     * Retrieve the parser registry used by this instance, which can be used to
581     * directly modify the parsing behavior.
582     * @return
583     */
584    public ParserRegistry getParsers() {
585        return parserRegistry;
586    }
587
588    public void setParsers( ParserRegistry parserRegistry ) {
589        this.parserRegistry = parserRegistry;
590    }
591
592    /**
593     * Get the default content-type used for parsing response data.
594     * @return a String or {@link ContentType} object.  Defaults to
595     * {@link ContentType#ANY}
596     */
597    public Object getContentType() {
598        return contentType;
599    }
600
601    /**
602     * Set the default content-type used to control response parsing and request
603     * serialization behavior.  If <code>null</code> is passed,
604     * {@link ContentType#ANY} will be used.  If this value is
605     * {@link ContentType#ANY}, the response <code>Content-Type</code> header is
606     * used to parse the response.
607     * @param ct a String or {@link ContentType} value.
608     */
609    public void setContentType( Object ct ) {
610        this.contentType = (ct == null) ? ContentType.ANY : ct;
611    }
612
613    /**
614     * Get the default content-type used to serialize the request data.
615     * @return
616     */
617    public Object getRequestContentType() {
618        return requestContentType;
619    }
620
621    /**
622     * Set the default content-type used to control request body serialization.
623     * If null, the {@link #getContentType() contentType property} is used.
624     * Additionally, if the <code>contentType</code> is {@link ContentType#ANY},
625     * a <code>requestContentType</code> <i>must</i> be specified when
626     * performing a POST or PUT request that sends request data.
627     * @param requestContentType String or {@link ContentType} value.
628     */
629    public void setRequestContentType( Object requestContentType ) {
630        this.requestContentType = requestContentType;
631    }
632}