fetching URL contents into a string using HttpWebRequest

Posted by Admin on May 13, 2009
103 comments

You can use the below class for fetching the contents of any HTTP URL using the HTTP GET method. This makes use the webrequest classes of .NET class library.

public static class GetUrl
{
    public static string FetchURL(string url)
   {

        const int bufSizeMax = 65536; // max read buffer size conserves memory
        const int bufSizeMin = 8192; // min size prevents numerous small reads
        StringBuilder sb;

        // A WebException is thrown if HTTP request fails
        try
        {

            // Create an HttpWebRequest using WebRequest.Create (see .NET docs)!
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            request.Method = WebRequestMethods.Http.Get;

            // Execute the request and obtain the response stream
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            Stream responseStream = response.GetResponseStream();

            // Content-Length header is not trustable, but makes a good hint.
            // Responses longer than int size will throw an exception here!
            int length = (int)response.ContentLength;

            // Use Content-Length if between bufSizeMax and bufSizeMin
            int bufSize = bufSizeMin;
            if (length > bufSize)
                bufSize = length > bufSizeMax ? bufSizeMax : length;

            // Allocate buffer and StringBuilder for reading response
            byte[] buf = new byte[bufSize];
            sb = new StringBuilder(bufSize);

            // Read response stream until end
            while ((length = responseStream.Read(buf, 0, buf.Length)) != 0)
                sb.Append(Encoding.UTF8.GetString(buf, 0, length));

            return sb.ToString();

        }
        catch (Exception ex)
        {
            sb = new StringBuilder(ex.Message);
            return sb.ToString();
        }

   }
}