利用者:Bcxfubot/北海道開発局5/prog

# [orig] request.py
# URL張替
# [http://www.as.hkd.mlit.go.jp/
# ↓
# [https://www.hkd.mlit.go.jp/as/
import re
import time
import pywikibot
import requests
from requests.exceptions import Timeout

target_re = "\[http:\/\/www\.[a-z][a-z]\.hkd\.mlit\.go\.jp\/ "
#max = 10
#max = 10
max = 200
sleepsec = 60

######################################################
# 処理モード
#procmode = 0
procmode = 1
######################################################


# http://www.as.hkd.mlit.go.jp/
# ↓
# https://www.hkd.mlit.go.jp/as/
def make_newurl(url):
    id = ""
    pattern = "http:\/\/www\.([a-z][a-z])\.hkd\.mlit\.go\.jp\/"
    m = re.search( pattern, url)
    if m:
        id = m.group(1)
    print("id="+id)
    if id == "":
        return ""
    newurl = "https://www.hkd.mlit.go.jp/" + id + "/"
    return newurl

# 指定されたURLの最終的なリダイレクト先を返す。
# エラーの場合は""を返す
def get_final_url(url):
    #response = requests.head(url)
    try:
        # PDFは重いのでheadにする 2020.1.13
        if ".pdf" in url:
            response = requests.head(url,timeout=2.0)
        else:
            response = requests.get(url,timeout=2.0)
    except Timeout:
        print("ERROR: timeout")
        raise
        return ""
    except Exception as e:
        print("ERROR: Exception")
        print(e)
        raise
        return ""
    print(response.status_code)
    print(response.url)
    #print(response.headers)
    if response.status_code != 200:
        return ""
    return response.url

# 指定されたhttpリンクがhttpsで同じパスで200ならfinalurlを返す
def check_200_https(url):
    print( "url=" + url )
    #httpsurl = url.replace( "http:" , "https:")
    #print( "httpsurl=" + httpsurl )
    finalurl = get_final_url( url ) 
    print( "finalurl=" + finalurl )
    if finalurl != "":
        if finalurl == url:
            return finalurl
    return ""


# ページを置換する
def replace_page(pagetitle):
    site = pywikibot.Site()
    page = pywikibot.Page(site, pagetitle)
    #text = page.text
    #print(text)
    linelist = page.text.split('\n')
    #print(linelist)

    comment_target_https = ""
    gaibu = 0
    modflag = 0
    outtext = ""
    for line in linelist:
        if (re.search("<ref",line) or
            re.search("ref>",line) or
            re.search("web.archive.org",line) or
            re.search("Wayback",line) ):
            outtext += line + "\n"
            continue
        #print(gaibu,line)
        #if "==外部リンク" in line:
        #if re.search("==[ ]*外部リンク",line):
        #    gaibu = 1
        #if gaibu == 1:
        #if target in line:
        if re.search(target_re,line):
            #print(gaibu,line)
            #line = line.replace( target, target_https)
            #print(gaibu,line)
            pattern = r"http://[^ \|\]\}<]+"
            matchedlist = re.findall( pattern, line)
            if matchedlist:
                for url in matchedlist:
                    print( "url=" + url )
                    newurl = make_newurl( url )
                    print( "newurl=" + newurl )
                    finalurl = check_200_https(newurl)
                    print( "finalurl=" + finalurl )
                    if finalurl != "" and finalurl != url:
                        line = line.replace( url, finalurl)
                        comment_target_https = comment_target_https + finalurl
                        print(gaibu,line)
                        modflag = 1
                        
        outtext += line + "\n"

    if modflag == 1:
        # 異常チェック：元のpage.textと新しいouttextの違いはhttpsだけなので
        # たかだか10バイトぐらいしか違わないはずである。
        # それ以上違う場合はなにかしらおかしいのでエラーとする
        difflen = len(outtext) - len(page.text)
        print("difflen=" + str(difflen))
        if ( ( difflen < 0 ) or
             ( difflen > 20 ) ):
            raise Exception
            exit(2)
        page.text = outtext
        #print(page.text)
        if procmode == 1:
            page.save("外部リンクの修正 " + comment_target_https + " ([[Wikipedia:Bot|Bot]]による編集)")

# 処理対象のページ名をひとつ返す
# 処理対象がない場合は""を返す
def get_pagetitle():
    path = "list"
    with open(path) as f:
        for s_line in f:
            s_line = s_line.rstrip("\n")
            #print(s_line)
            #if not re.search(",sumi", s_line):
            if not s_line.endswith(",sumi"):
                return s_line
    return ""

# 処理した行にsumiをつける
def done_pagetitle(pagetitle):
    path = "list"
    alltext = ""
    with open(path) as f:
        for s_line in f:
            s_line = s_line.rstrip("\n")
            #print(s_line + "\n")
            #if re.search(pagetitle, s_line):
            if pagetitle == s_line:
                s_line = s_line + ",sumi"
            alltext += s_line + "\n"
    with open(path, mode='w') as f:
        f.write(alltext)
    return ""

def sub():
    num = 0
    for i in range(max):
        num = num + 1
        pagetitle = get_pagetitle()
        print("[" + str(num) + "/" + str(max) + "]" + ":" + "pagetitle=" + pagetitle)
        if pagetitle == "":
            break
        replace_page(pagetitle)
        done_pagetitle(pagetitle)
        
        if ( i < (max - 1) ):
            print("sleep(" + str(sleepsec) + ")")
            time.sleep(sleepsec)

def main():
    sub()
    print("done.")

if __name__ == '__main__':
    main()