#!/usr/bin/python
# pylint: disable=consider-using-f-string
"""
[orig] first_archive.py
first_archive.pyは、新聞ニュース記事などで使う。最初のアーカイブが重要なやつ。
[http://www.blog.com/blog.html ページタイトル]
↓
{{Wayback|url=http://www.blog.com/blog.html |title=ページタイトル}}
一行に複数リンクがある場合に、最後のやつだけしか置換されない問題あり
→ 後日再度本スクリプトを動かして、最後以外のも置換させる必要あり。2020.2.13
"""
import re
import time
import urllib.parse
from urllib.parse import urlparse
import pywikibot
import requests
from requests.exceptions import Timeout
TARGET_RE = r"https:\/\/www\.toei\.co\.jp\/release\/tv\/"
MAXCOUNT = 1
SLEEPSEC = 60
######################################################
# 処理モード
PROCMODE = 0
#PROCMODE = 1
######################################################
def get_domain(target):
""" ドメインを取得する """
url = ""
result = re.search ( "(http[^ ]+)", target)
if result:
url = result.group(1)
else:
return target
parsed_uri = urlparse(url )
result = '{uri.netloc}'.format(uri=parsed_uri)
return result
def get_date_core(origurl):
""" 日付取得処理core """
encoded_url = urllib.parse.quote(origurl, safe="")
print("encoded_url = "+ encoded_url)
#spark_url = "https://web.archive.org/__wb/sparkline?url=" + encoded_url +
# "&collection=web&output=json"
#api_url = "https://archive.org/wayback/available?url=" + encoded_url + "×tamp=20010101"
api_url = "https://archive.org/wayback/available?url=" + encoded_url + "×tamp=21010101"
#print("spark_url = "+ spark_url)
print("api_url = "+ api_url)
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0'
headers = {'User-Agent': user_agent}
try:
#response = requests.get(spark_url,timeout=60.0)
response = requests.get(api_url,timeout=60.0,headers=headers)
except Timeout:
print("ERROR: timeout")
raise
except Exception as err:
print("ERROR: Exception")
print(err)
raise
print("response.text = " + response.text)
#if "\"first_ts\":null" in response.text:
# return ""
if "504 Gateway Time-out" in response.text:
return ""
data = response.json()
print(data)
#lastdate = data["last_ts"]
#print(lastdate)
#return lastdate
#firstdate = data["first_ts"]
try:
firstdate = data["archived_snapshots"]["closest"]["timestamp"]
except TypeError:
firstdate = ""
except KeyError:
firstdate = ""
print(firstdate)
return firstdate
def get_date(origurl):
""" 日付取得処理 """
result = ""
for i in range(2):
result = get_date_core(origurl)
if result != "":
break
print("sleep(10), i=" + str(i))
time.sleep(10)
return result
def get_stat( url ):
""" status取得処理 """
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0'
headers = {'User-Agent': user_agent}
try:
if ".pdf" in url:
response = requests.head(url,headers=headers,timeout=5.0)
else:
response = requests.get(url,headers=headers,timeout=5.0)
except requests.exceptions.ConnectionError:
print("ERROR: ConnectionError")
return 404
except Timeout:
print("ERROR: timeout")
raise
except Exception as err:
print("ERROR: Exception")
print(err)
raise
print(response.status_code)
print(response.url)
return response.status_code
def make_newline( origline ):
""" 新行を作成する """
#print("make_newline(): origline=" + origline)
newline = ""
#result = re.search( "^(.*)\[(http:\/\/dir\.yahoo\.co\.jp\/[^ ]+) ([^\]]*)\](.*)$" , origline )
#result = re.search( TARGET_RE , origline )
result = re.search( r"^(.*)\[(" + TARGET_RE + r"[^ ]*)[ ]+([^\]]*)\](.*)$", origline)
if result:
pre = result.group(1)
print("pre="+pre)
origurl = result.group(2)
origtext = result.group(3)
post = result.group(4)
print("origurl = " + origurl)
print("origtext = " + origtext)
# origtextに[[]]内部リンクがある場合はうまく処理できないのでここではじく2020.7.24
result2 = re.search( r"\[\[", origtext)
if result2:
return origline
#stat = get_stat( origurl )
#if stat not in ( 400, 404, 405, 410):
# return origline
date = get_date( origurl )
#if date == "":
# return ""
if date == "":
#date = "*"
# アーカイブがない場合は修正しないことにした。2021.10.15
return origline
print("date = " + date)
origtext = origtext.replace("|", "|")
newline = pre + "{{Wayback|url=" + origurl + \
" |title=" + origtext + \
" |date=" + date + "}}" + post
print("newline = " + newline)
return newline
def replace_page(site,pagetitle):
""" ページを更新する """
is_saved = False
page = pywikibot.Page(site, pagetitle)
#text = page.text
#print(text)
linelist = page.text.split('\n')
#print(linelist)
gaibu = 0
modflag = 0
outtext = ""
for line in linelist:
if (re.search("<ref",line) or
re.search("ref>",line) or
re.search("web.archive.org",line) or
re.search("Wayback",line) ):
outtext += line + "\n"
continue
#print(gaibu,line)
result = re.search( r"^\*", line)
if result:
#if target in line:
if re.search( TARGET_RE, line):
newline = make_newline( line )
if newline != "":
if line != newline:
line = newline
print(gaibu,line)
modflag = 1
outtext += line + "\n"
# 最終行の改行の調整
if page.text[-1:] != "\n":
if outtext[-1:] == "\n":
outtext = outtext[:-1]
if modflag == 1:
page.text = outtext
if PROCMODE == 1:
#page.save("外部リンクの修正 http:// -> https:// ([[Wikipedia:Bot|Bot]]による編集)")
#page.save("外部リンクの修正 http:// -> {{Wayback}} ([[Wikipedia:Bot|Bot]]による編集)")
page.save("外部リンクの修正 http:// -> {{Wayback}} (" +
get_domain( TARGET_RE.replace("\\","") ) +
") [[Wikipedia:Bot作業依頼#東映のリンク切れをウェイバックマシンに置換]] ([[Wikipedia:Bot|Bot]]による編集)")
is_saved = True
return is_saved
# 処理対象のページ名をひとつ返す
# 処理対象がない場合は""を返す
def get_pagetitle():
""" 処理対象のページ名をひとつ返す """
path = "list"
with open(path, encoding="utf-8") as file:
for s_line in file:
s_line = s_line.rstrip("\n")
#print(s_line)
#if not re.search(",sumi", s_line):
if not s_line.endswith(",sumi"):
return s_line
return ""
# 処理した行にsumiをつける
def done_pagetitle(pagetitle):
""" 処理した行にsumiをつける """
path = "list"
alltext = ""
with open(path, encoding="utf-8") as file:
for s_line in file:
s_line = s_line.rstrip("\n")
#print(s_line + "\n")
#if re.search(pagetitle, s_line):
if pagetitle == s_line:
s_line = s_line + ",sumi"
alltext += s_line + "\n"
with open(path, mode='w', encoding="utf-8") as file:
file.write(alltext)
return ""
def sub():
""" sub """
site = pywikibot.Site()
site.login()
for i in range(MAXCOUNT):
pagetitle = get_pagetitle()
print("[" + str(i + 1) + "/" + str(MAXCOUNT) + "]" + ":" + "pagetitle=" + pagetitle)
if pagetitle == "":
break
is_saved = replace_page(site,pagetitle)
done_pagetitle(pagetitle)
#if ( i < (MAXCOUNT - 1) ):
# print("sleep(" + str(SLEEPSEC) + ")")
# time.sleep(SLEEPSEC)
print("is_saved=" + str(is_saved))
if is_saved:
print("sleep(" + str(SLEEPSEC) + ")")
time.sleep(SLEEPSEC)
else:
print("sleep(" + str(5) + ")")
time.sleep(5)
def main():
""" main """
sub()
print("done.")
if __name__ == '__main__':
main()
