帮我加快这段代码的速度-Python

如何解决帮我加快这段代码的速度-Python

| 伙计们，我正在编写此程序，该程序会处理一系列推文并返回使用最多的单词。 我想使其更快，但是我想知道您是否可以帮助指出一些我可以提高速度的问题或领域。 谢谢 见下面的代码

#import string
import re
from string import punctuation
from operator import itemgetter
import pprint


class Tweet:
    def __init__(self,timestamp,userId,message):
        self.timestamp = timestamp
        self.userId = userId
        self.message = message

    def getDate(self):
        tokens = re.split(\' \',self.timestamp)
        return tokens[0]

    def __repr__(self):
        return \"[timestamp=%s userId=%s message=%s]\" % (self.timestamp,self.userId,self.message)

outfile  = file 
def readOneTweet(file):

    \"\"\" Reads a single tweet from the file,and returns the string containing the tweet.
    This will often just be a single line from the file,but may be more if it ends with a slash.
    \"\"\"
    lineBuffer = \"\" 
    while True:
        # Todo: read the line and strip it

        rawLine = file.readline().strip(\'\\n\')
        if (len(rawLine)== 0):
            break

        lineBuffer +=rawLine

        if (rawLine[(len(rawLine)-1)]!= \"\\\\\"):
            break
    return lineBuffer 



def readTweets():
    tweets = []
    inputfile = raw_input(\"Enter filename: \")

    # move the try / except around a single tweet.
    # so that we can keep going if we encounter a line with an error.
    try:
        f = open(inputfile,\"r\")

        while True:
            tweet = readOneTweet(f) # readOneTweet is method
            if not tweet:
                break
            try:
                linestrip = tweet.rstrip()

                split_word = re.split(\'\\t\',linestrip.lower()) #(\'/([^a-z])([A-Z]) ([0-9])/n:.;\\]+/\',line.lower())

                tweetTime = split_word[1]
                userId = split_word[0]
                message = split_word[2]
                tweets.append(Tweet(tweetTime,message))
                if len(tweets) % 10000 == 0:
                    print \'read\',len(tweets),\'tweets\'
            except IndexError,e:
                print \"bad tweet\",tweet
    except IOError: 
        print \"file not found!\"
    return tweets

######################DATA ##############
\"\"\"
- Need to separate tweets
- Obtain information about each tweet - UserID,Time,words
\"\"\"

def writeWordFile(word):
    toWrite = \'test.txt\'
    fileHandle = open ( toWrite,\'w\' )
    for i in word:
        fileHandle.write (i)

def dailyMessages(twt):
    dailyMsg =dict ()
    for i in twt:
        date =i.getDate()
        #print  i.message
        #dailyMsg[date] =messageList
        if dailyMsg.has_key(date):
            dailyMsg[date].append(twt)
        else:
            dailyMsg[date] =[twt]
    #for k,v in dailyMsg.items():
        #print k,v,\'\\n\'
    return dailyMsg    

\"\"\"
Takes dailyTweets and perform  word coun. 
\"\"\"
def dailyWord(tweetsByDay):
    dailyTweetsWordCount = { }
    for date in tweetsByDay.keys():
        dayTweets =tweetsByDay[date]
        if len(dayTweets) != 0:
            count = wordCount(dayTweets)
            dailyTweetsWordCount[date] = count
    return dailyTweetsWordCount


def wordCount(tweets):
    \"\"\"Takes a list of tweets and returns a dictionary of counts for words\"\"\"
    N = 100
    # we\'ll build a dictionary mapping each word to a SET of users who have used it
    wordTweeters = {}
    for tweet in tweets:
       # print tweet
        for i in tweet:
            for word in i.message.split():
                if not  wordTweeters.has_key(word):
                    wordTweeters[word] = set()
                wordTweeters[word].add(i.userId)

    # we\'ll build up a dictionary mapping each word to the number of users who have used it.
    p = dict ()
    #print wordTweeters
    for day in wordTweeters.keys():
        usersNo = len (wordTweeters[day])
        p[day] = usersNo
    #print wordTweeters

    return  p  #wordTweeters,p

def searchForMemes(tweetUserCounts):
    for  key in tweetsUserCounts.keys():
       # for pmeme in tweetUserCounts
       pass


    \"\"\"Takes information returned by daily word\"\"\"


def isMeme(word,day1Count,day2Count,day3Count):

    #takes the daily count
    # check if it is a meme
    #First -  check count
        #check count in different days
        # determine the if it qualifies as a tweet
        # if not drop it  do not do below checks 
    #Second - time stamp
        #CHECK ITS TIME TRACK
        #How is the count of over time
        # rise and fall
        # 
    #Third - user id
        # check if is form different users
            #how many of those counts are from different users
       pass 

def dayUserCount(z,word,d1,d2,d3):
    \"\"\" assume dictionary will be input\"\"\"

    # check if the word exist in the dictionary

    if z.has_key(d1):
        date1 =z[d1]
        #print value.keys()
        if  date1.has_key(word):
            print date1
            c1 =date1[word]
        else:
            print \"word not used in %s\"%d1
            c1 =0
    else:
        print \'date does not exist\'

    if z.has_key(d2):
        #print value.keys()
        date2 =z[d2]
        if  date2.has_key(word):
            print date2
            c2 =date2[word]
        else:
            print \"word not used in %s\"%d2
            c2 =0
    else:
        print \'date does not exist\'

    if z.has_key(d3):
        date3 = z[d3]
        if date3.has_key(word):
            print date3
            c3 =date3[word]
        else:
            print \"word not used in %s\" %d3
            c3 =0
    else:
        print \'date does not exist\'

    result = \"Word: %s,%s count: %s,%s count: %s\"%(word,c1,c2,d3,c3)
    return result           



# supportive functions 
def hashtag(tw):
    hasht =[]
    for word in tw.split():
        if word.startswith(\'#\'):
            hasht.append(word)
    return hasht


def httpTag(tw):
    http =[]
    for word in tw.split():
        if word.startswith(\'http\'):
            http.append(word)
    return http

def reply(tw):
    reply =[]
    for word in tw.split():
        if word.startswith(\'@\'):
            reply.append(word)
    return reply

def reTweet(tw):
    rt =[]
    for word in tw.split():
        if word.startswith(\'rt\') or word.startswith(\'RT\'):
            rt.append(word)
    return rt






\"\"\"
Old functions 
\"\"\"
def writetoFile(tweet):
    #filename = test.txt
    filek = open(\'test.txt\',\'w\')
    print \"writing on the file: \"
    filek.write(tweet)
   # print \" _____--------______\" + tweet
    filek.close()

# count word frequency.
def f2count():
    N = 100000000000
    words = {}
    words_gen = (word.strip(punctuation).lower()
                 for line in open(\'c.txt\')
                        for word in line.split())
    for word in words_gen:
        words[word] = words.get(word,0) + 1

    top_words = sorted(words.iteritems(),key=lambda(word,count): (-count,word))[:N] 

    for word,frequency in top_words:
        print \"%s: %d\" % (word,frequency)

    

解决方法

if (len(rawline) == 0):

可以写成

if rawline:

切勿使用

len(rawline) - 1

作为索引，而应使用

rawline[-1]

。 我不知道为什么你只用6英镑就用5英镑。 不要使用

dailyMsg.has_key(date)

，请使用

date in dailyMsg

。 当您遍历

tweetsByDay

时，您确实应该这样做：

for date,value in tweetsByDay.items():`

这样，您不必手动将值绑定到键。 那仅仅是一个开始。还有更多的问题需要解决。我认为您真的只需要精通Python即可-从阅读代码中可以清楚地了解到，要么Python不是您的第一语言，要么您是从没有教您如何编写好它的资源中学到的。例如，为什么要在条件语句两边加上括号？在Python中这不是必需的（尽管这是来自类似Algol的语言（如C或Java）的工件。为什么用

dict()

代替

{}

？最好用第二种方式写一个空的字典。您可能会发现有关惯用Python的本教程很有帮助。     ,

wordCount()

可以并联运行。由于每个推文都不直接依赖于另一个推文，因此没有理由依次遍历该列表。将推文列表分成较小的列表，然后在每个子列表上创建一个线程。他们的子词典创建完成后，您可以做一些工作将它们全部合并为一个词典。 编辑： 有关如何使列表求和并行化的示例。您将更改线程的主体以执行任何任务。

from threading import Thread

numbers = range(1000)

class Sum(Thread):
  def __init__(self,numList):
    Thread.__init__(self)
    self.numList = numList
    self.total = 0

  def run(self):
    for num in self.numList:
      self.total += num

numThreads = 7
threads = []
perThread = len(numbers)/numThreads
for i in xrange(numThreads):
  start = i*perThread
  t = Sum(numbers[start:len(numbers) if i == numThreads-1 else start+perThread])
  t.start()
  threads.append(t)

grandTotal = 0
for t in threads:
  t.join()
  grandTotal += t.total

print grandTotal

    ,此代码充满了未优化的代码段。 例如，每次调用函数都需要时间。不要对函数进行一些无用的调用，您将节省时间。 某些调用可以用理解列表代替：hashtag，httpTag等 我可以帮助优化，但是： 1-我目前没有足够的时间进行此类长期工作 2-我们无法优化，因为代码不完整：以下函数在哪里？ ：

readTweets
writeWordFile
dailyMessages
dailyWord
wordCount
searchForMemes
isMeme
dayUserCount
hashtag
httpTag
reply
reTweet
writeToFile
f2count

3-我很想回答新注册的人员，这些人员在stackoverflow上遇到了沉重的问题，然后又消失了，有时甚至不发表任何新闻或发表评论。打扰一下，如果你不想这么做的话 编辑 writeToFile f2count 必须从清单中删除。