HTTP Class for Python 3.1

[Riddle]

HTTP.py

#!/usr/bin/env python

#HTTP Class
#Author: Riddle
#E-mail: riddle_lg@live.com

import General
import urllib.request
import urllib.parse
import gzip
import http.cookiejar
import socket
import io

socket.setdefaulttimeout(10)

class HTTP:

    def __init__(self):
        self.USER_AGENT = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 (.NET CLR 3.5.30729)'
        self.ENCODING = 'gzip'
        
        self.ID = None
        self.Cookie_Jar = http.cookiejar.LWPCookieJar()
        
        self.Last_URL = None
        self.Referer = None

    def Set_Referer (self, Ref = None):
        self.Referer = Ref

    def Clear_Cookies (self):
        self.Cookie_Jar.clear_session_cookies()

    def Get_Opener (self):
        Opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.Cookie_Jar))

        if self.Referer != None and self.ENCODING != None:
            Opener.addheaders = [('User-Agent', self.USER_AGENT),('Accept-Encoding', self.ENCODING),('Referer', self.Referer)]
        elif self.Referer != None and self.ENCODING == None:
            Opener.addheaders = [('User-Agent', self.USER_AGENT),('Referer', self.Referer)]
        elif self.Referer == None and self.ENCODING != None:
            Opener.addheaders = [('User-Agent', self.USER_AGENT), ('Accept-Encoding', self.ENCODING)]
        else:
            Opener.addheaders = [('User-Agent', self.USER_AGENT)]

        self.Referer = None

        return Opener

    def POST (self, URL, Data):
        if Data.__class__ == dict:
            Data = urllib.parse.urlencode(Data)

        return self.GET(URL, Data)

    def GET (self, URL, Data = None, enc = "utf-8"):
        try:
            Request = urllib.request.Request(URL, Data)
            Opener = self.Get_Opener()
            Contents = Opener.open(Request)
            Source = Contents.read()
            if self.ENCODING == 'gzip':
                try:
                    Compressed_Data = io.BytesIO(Source)
                    Decompressed_Data = gzip.GzipFile(fileobj = Compressed_Data)
                    Source = Decompressed_Data.read()

                except:
                    General.General().Log_Error("Error decoding GZip.\nURL: " + URL + "\nData: " + str(Data) + "\n\n")
                    
            return str(Source,enc)

        except:
            return "__ERROR__"

    def Get_Last_URL(self):
        return self.Last_URL

    def Set_Last_URL(self, URL = None):
        self.Last_URL = URL

if __name__ == "__main__":
    print(HTTP().GET("http://www.google.com"))

General.py

#!/usr/bin/env python

#####################
#Author: Riddle
#Website: www.logicalgamers.com
#E-mail: riddle_lg@live.com
#####################

#Collection of useful and general functions

import hashlib
import sys
import re
import os
import time
import random

class General:
    def __init__(self):
        random.seed(time.time())
        pass

    def sleep(self,s):
        time.sleep(s)

    def random(self,low,high):
        return random.randrange(low,high)
    
    def md5Digest(self, Str1):        
        a = hashlib.md5()   #Create new MD5 object
        a.update(Str1.encode('utf-8'))  #Prepares password string to be digested
    
        return a.hexdigest()    #Returns the final outcome

    def Log_Error(self, Data, File_Name = 'Error Log.txt', Mode = 'a'):
        Target = open(os.getcwd()+self.sep()+File_Name, Mode)
        Target.write(Data)
        Target.close()

    def Paused_Exit(self):
        a = input("\nPress Enter To Exit...")
        sys.exit(1)

    def Str_To_File(self, str1, path, mode = "a"):
        f = open(path, mode)
        f.write(str1)
        f.close()

    def List_To_File(self, lst1, path, mode = "a", line_break = False): #Line Break adds a \n after each line
        f = open(path,mode)

        if line_break == False:
            f.writelines(lst1)
        else:
            for line in lst1:
                f.write(line+"\n")
                
        f.close()

    def File_To_List(self, path):
        f = open(path, "r")
        lines = f.readlines()
        new_list = []
        
        for line in lines:
            new_list.append(line.strip())

        f.close()
        return new_list

    def File_To_Str(self, path):
        f = open(path, "r")
        str1 = f.read()
        f.close()

        return str1

    def De_Dupe_List(self, lst, case_sensitive = False):

        if case_sensitive == False:
            for item in lst:
                for citem in lst:
                    if item != citem:
                        if item.lower() == citem.lower():
                            lst.remove(citem)
                            
        for item in lst:
            lst.remove(item)
        
            if not item in lst:
                lst.append(item)

        return lst

    def Find_All(self, pattern, strToSearch):
        #Find all matching strings with a Reg Ex.
        pattern = re.compile(pattern)
        return re.findall(pattern, strToSearch)

    def Dir_Exists(self,path):
        "Returns True if it exists"
        return os.path.isdir(path)

    def File_Exists(self,path):
        "Returns True if the file exists"
        return os.path.isfile(path)

    def Cur_Dir(self):
        "Returns Current Directory"
        return os.getcwd()

    def mkdir(self,path):
        "Makes Directory on Path"
        os.mkdir(path)

    def sep(self):
        return os.path.sep

I'll post some examples later.

Functions to use: GET, POST, Set_Referer, Clear_Cookies, Set_Last_URL, Get_Last_URL.

[Nat]

O very nice Riddle.

Might hafta use this.

[Riddle]

Go for it. I'll be documenting it sometime during this/next week.

[MattSmith]

If i had time to get into python again i would but im way to busy nowadays -.-


Nice / thanks for release.

[takoyaki]

Sorry for a likely stupid question, but when i run this i get an error, i believe it says something along the lines of
"Syntax error general does not exist" or something similar.
and a similar error for urllib.request and parse
are these things I must personally create? because I thought they existed already within python.
Thanks for your time Riddle & anyone else who replies.

EDIT: while looking online i found something importing urllib.parse like this:
">>> from urllib.parse import urlparse" would that be an acceptable way to do it? or would this not work?

[Riddle]

I forgot to include that. It's a class I created that has some useful stuff.

[Riddle]

Sorry for a likely stupid question, but when i run this i get an error, i believe it says something along the lines of
"Syntax error general does not exist" or something similar.
and a similar error for urllib.request and parse
are these things I must personally create? because I thought they existed already within python.
Thanks for your time Riddle & anyone else who replies.

EDIT: while looking online i found something importing urllib.parse like this:
">>> from urllib.parse import urlparse" would that be an acceptable way to do it? or would this not work?

I'm not sure. Probably not. It might of been an old version of Python. Or it's a different module.

[takoyaki]

I'm not sure. Probably not. It might of been an old version of Python. Or it's a different module.

hmm, Then how would I get mine to import those? I checked the manual and it says that they exist, until I try import.
Thanks

[Riddle]

If you have Python 3.1 installed, then you simply save HTTP.py and General.py in the same folder. You run HTTP.py to see the Google.com test. It should work.

[Chad]

Don't forget to import it. Not sure if you need that in python.

[Riddle]

If you just run the HTTP.py script, it has all the imports needed already there. ;D

If you're trying to access HTTP from another script, you DO need to import it, and, for now, they should be in the same folder (until you learn more importing methods).

[MattSmith]

Bringing back incrediold thread but i need this class for 2.7, got it? Or something like it? D: Alex says 2.7 is better so im going to do that,

[Riddle]

This might be it, didn't test. It might also not be as complete as the one in the first post... I'm sure you can add stuff to it if you need it.

Also, 2.7 is not "better" than 3.X. 2.7 just happens to be compatible with other resources out there.

#!/usr/bin/env python

# Author: Riddle
# Email: REMOVED
# Date: 3/13/2011

# HTTPWrapper simulates a HTTP client to connect with webservers and retrieve data.

#Example of use at end of file.

#Might want to add cookie management, FTP support, file download support, etc. in the future?

import urllib
import urllib2
import gzip
import StringIO
import cookielib
import sys

class HTTPWrapper:
    def __init__(self, user_agent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.44 Safari/534.7"):
        self.user_agent = user_agent
        self.cjar = cookielib.LWPCookieJar()
        self.referer = None
        self.extra_headers = []

    def Set_Referer(self, ref = None):
        self.referer = ref

    def Add_Headers(self, headers):
        self.extra_headers = headers

    def Get_Opener(self):
        # Assume that websites are GZip enabled
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cjar))
        headers = [('User-Agent', self.user_agent),('Accept-Encoding', 'gzip,deflate')]

        if self.referer != None:
            headers.append(('Referer', self.referer))

        for x in self.extra_headers:
            headers.append(x)
            
        opener.addheaders = headers
        self.referer = None
        return opener

    def POST(self, url, data):
        #Returns None on error!
        #data is a dictionary of parameters
        if data.__class__ == dict:
            data = urllib.urlencode(data)
        return self.GET(url, data)

    def GET(self, url, data = None, enc = "utf-8"):
        #Returns None on error!

        if url.lower().find("http://") < 0:
            url = "http://" + url
            
        try:
            req = urllib2.Request(url, data)
            opener = self.Get_Opener()
            contents = opener.open(req)
            source = contents.read()

            try:
                comp_data = StringIO.StringIO(source)
                dec_data = gzip.GzipFile(fileobj = comp_data)
                source = dec_data.read()

            except: #Log the error?
                pass
                #print "\n\nHTTPWrapper_ERROR: GZip did not work!\n\n"
                                
            return source.decode(enc)

        except:
            return None

###################################################
# Example Use:                                    #
# A login request to www.mysite.com (Not tested)  #
###################################################
# pdata = {'username':'Admin', 'password':'GOD'}  #
# http = HTTPWrapper()                            #
# http.Set_Referer("http://www.mysite.com")       #
# html = http.POST("http://www.mysite.com", pdata)#
###################################################

[MattSmith]

Awesome! Thanks riddle, works perfectly. Ill add what i need and ill repost here when I feel its worth posting.

[Artificial]

Also, 2.7 is not "better" than 3.X. 2.7 just happens to be compatible with other resources out there.

And that's exactly what I said Matthew. Tsk, tsk.

[MattSmith]

And that's exactly what I said Matthew. Tsk, tsk.

>.> which one should i use? We need to set an official LG version so users can run everything we make.

[Artificial]

I've already told you I'm using 2.6 because a lot of Python libraries aren't forward compatible. 2.7 should be fine.

[Riddle]

Alex is right. Just stick to 2.7 and you shouldn't have much problems with what is out there already.

[MattSmith]

Alright, sounds good. Ima stick with 2.7 then. Thanks again