Ranking of the URL's by tf idf added
This commit is contained in:
parent
8494af6d6d
commit
5acf243701
184
App.java
184
App.java
@ -1,184 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Scanner;
|
|
||||||
import java.util.StringTokenizer;
|
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
import org.jsoup.select.Elements;
|
|
||||||
|
|
||||||
public class App {
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
String url[] = {"https://www.thesun.co.uk/?s=", "https://www.bbc.co.uk/search?q=", "https://www.skysports.com/search?q="};
|
|
||||||
//ArrayList<String> key = new ArrayList<String>();
|
|
||||||
Article data = new Article();
|
|
||||||
//key.add("salah");
|
|
||||||
//key.add("Liverpool");
|
|
||||||
//key.add("Champions league");
|
|
||||||
// 가져오고 싶은 정보가 있는 웹페이지의 url
|
|
||||||
Document doc = null;
|
|
||||||
Elements element = null;
|
|
||||||
|
|
||||||
Scanner scanner = new Scanner(System.in);
|
|
||||||
System.out.print("Please type keywords : ");
|
|
||||||
String key[] = scanner.nextLine().split(",");
|
|
||||||
scanner.close();
|
|
||||||
for (int j = 0; j < url.length; j++) {
|
|
||||||
for (int i = 0; i < key.length; i++) {
|
|
||||||
String urlTmp = url[j] + key[i];
|
|
||||||
doc = Jsoup.connect(urlTmp).execute().parse(); // Document에 url 페이지의 데이터를 가져온다.
|
|
||||||
/* } catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}*/
|
|
||||||
if (url[j].equals("https://www.thesun.co.uk/?s=")) {
|
|
||||||
element = doc.select("div.search-results-wrap");
|
|
||||||
for (Element el : element.select(".teaser-item")) {
|
|
||||||
if (el.select("p").text().toLowerCase().contains(key[i].toLowerCase())) {
|
|
||||||
data.setHeadline(el.select("p").text());
|
|
||||||
Elements elUrl = el.select(".teaser__copy-container a");
|
|
||||||
data.setUrl(elUrl.first().absUrl("href"));
|
|
||||||
String temp = el.select(".search-date").text();
|
|
||||||
data.setDate(changeDate(temp));
|
|
||||||
data.setSite("The Sun");
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (url[j].equals("https://www.bbc.co.uk/search?q=")) {
|
|
||||||
element = doc.select("section.search-content");
|
|
||||||
for (Element el : element.select("li[data-result-number]")) {
|
|
||||||
data.setHeadline(el.select("h1").select("a").text());
|
|
||||||
Elements elUrl = el.select("a[href]");
|
|
||||||
data.setUrl(elUrl.first().absUrl("href"));
|
|
||||||
String temp = el.select(".display-date").text();
|
|
||||||
data.setDate(changeDate(temp));
|
|
||||||
data.setSite("BBC");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (url[j].equals("https://www.skysports.com/search?q=")) {
|
|
||||||
element = doc.select("div.news-list");
|
|
||||||
for (Element el : element.select("div.news-list__item")) {
|
|
||||||
data.setHeadline(el.select("h4").select("a").text());
|
|
||||||
Elements elUrl = el.select("a[href]");
|
|
||||||
data.setUrl(elUrl.first().absUrl("href"));
|
|
||||||
String temp = el.select(".label__timestamp").text();
|
|
||||||
data.setDate(changeDate2(temp));
|
|
||||||
data.setSite("SKYSPORTS");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
String content = "";
|
|
||||||
for(int i = 0; i < data.getHowManyData(); i++) {
|
|
||||||
content = "";
|
|
||||||
doc = Jsoup.connect(data.getUrl(i)).execute().parse();
|
|
||||||
if (data.getSite(i) == "The Sun") {
|
|
||||||
element = doc.select("div.article__content");
|
|
||||||
for (Element el : element.select("p")) {
|
|
||||||
content += el.text();
|
|
||||||
}
|
|
||||||
data.setContent(content);
|
|
||||||
}
|
|
||||||
else if (data.getSite(i) == "BBC") {
|
|
||||||
element = doc.select("div#story-body");
|
|
||||||
for (Element el : element.select("p")) {
|
|
||||||
content += el.text();
|
|
||||||
}
|
|
||||||
data.setContent(content);
|
|
||||||
}
|
|
||||||
else if (data.getSite(i) == "SKYSPORTS") {
|
|
||||||
element = doc.select("div.article__body");
|
|
||||||
for (Element el : element.select("p")) {
|
|
||||||
if (!el.hasClass("widge-marketing__text")) {
|
|
||||||
content += el.text();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
data.setContent(content);
|
|
||||||
}
|
|
||||||
System.out.println(data.getDate(i));
|
|
||||||
System.out.println(data.getHeadline(i));
|
|
||||||
System.out.println(data.getUrl(i));
|
|
||||||
System.out.println(data.getSite(i));
|
|
||||||
System.out.println(data.getContent(i));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int changeDate2(String date) {
|
|
||||||
date = date.substring(0, 2) + date.substring(2 + 1);
|
|
||||||
date = date.substring(0, 4) + date.substring(4 + 1);
|
|
||||||
String year = date.substring(4, 8);
|
|
||||||
String month = date.substring(2,4);
|
|
||||||
String day = date.substring(0, 2);
|
|
||||||
String fdate = year + month + day;
|
|
||||||
|
|
||||||
int mydate = Integer.parseInt(fdate);
|
|
||||||
return mydate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int changeDate(String date) {
|
|
||||||
int formdate = 0;
|
|
||||||
String sp[] = date.split(" ");
|
|
||||||
formdate += Integer.parseInt(sp[2]) * 10000;
|
|
||||||
formdate += Integer.parseInt(sp[0]);
|
|
||||||
|
|
||||||
switch (sp[1]) {
|
|
||||||
case "January":
|
|
||||||
case "Jan":
|
|
||||||
formdate += 100;
|
|
||||||
break;
|
|
||||||
case "February":
|
|
||||||
case "Feb":
|
|
||||||
formdate += 200;
|
|
||||||
break;
|
|
||||||
case "March":
|
|
||||||
case "Mar":
|
|
||||||
formdate += 300;
|
|
||||||
break;
|
|
||||||
case "April":
|
|
||||||
case "Apr":
|
|
||||||
formdate += 400;
|
|
||||||
break;
|
|
||||||
case "May":
|
|
||||||
formdate += 500;
|
|
||||||
break;
|
|
||||||
case "June":
|
|
||||||
case "Jun":
|
|
||||||
formdate += 600;
|
|
||||||
break;
|
|
||||||
case "July":
|
|
||||||
case "Jul":
|
|
||||||
formdate += 700;
|
|
||||||
break;
|
|
||||||
case "August":
|
|
||||||
case "Aug":
|
|
||||||
formdate += 800;
|
|
||||||
break;
|
|
||||||
case "September":
|
|
||||||
case "Sep":
|
|
||||||
formdate += 900;
|
|
||||||
break;
|
|
||||||
case "October":
|
|
||||||
case "Oct":
|
|
||||||
formdate += 1000;
|
|
||||||
break;
|
|
||||||
case "November":
|
|
||||||
case "Nov":
|
|
||||||
formdate += 1100;
|
|
||||||
break;
|
|
||||||
case "December":
|
|
||||||
case "Dec":
|
|
||||||
formdate += 1200;
|
|
||||||
break;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return formdate;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
63
Article.java
63
Article.java
@ -1,63 +0,0 @@
|
|||||||
import java.util.ArrayList;
|
|
||||||
|
|
||||||
public class Article {
|
|
||||||
|
|
||||||
public Article() {
|
|
||||||
date = new ArrayList<Integer>();
|
|
||||||
Headline = new ArrayList<String>();
|
|
||||||
Url = new ArrayList<String>();
|
|
||||||
Site = new ArrayList<String>();
|
|
||||||
Content = new ArrayList<String>();
|
|
||||||
}
|
|
||||||
|
|
||||||
private ArrayList<Integer> date;
|
|
||||||
private ArrayList<String> Headline;
|
|
||||||
private ArrayList<String> Url;
|
|
||||||
private ArrayList<String> Site;
|
|
||||||
private ArrayList<String> Content;
|
|
||||||
|
|
||||||
public void setDate(int num) {
|
|
||||||
date.add(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setHeadline(String head) {
|
|
||||||
Headline.add(head);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setUrl(String url) {
|
|
||||||
Url.add(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSite(String site) {
|
|
||||||
Site.add(site);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setContent(String content) {
|
|
||||||
Content.add(content);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public int getHowManyData() {
|
|
||||||
return Headline.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getDate(int num) {
|
|
||||||
return date.get(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getHeadline(int num) {
|
|
||||||
return Headline.get(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getUrl(int num) {
|
|
||||||
return Url.get(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getSite(int num) {
|
|
||||||
return Site.get(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getContent(int num) {
|
|
||||||
return Content.get(num);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -11,8 +11,7 @@
|
|||||||
<option name="LAST_RESOLUTION" value="IGNORE" />
|
<option name="LAST_RESOLUTION" value="IGNORE" />
|
||||||
</component>
|
</component>
|
||||||
<component name="DefaultGradleProjectSettings">
|
<component name="DefaultGradleProjectSettings">
|
||||||
<option name="testRunner" value="GRADLE" />
|
<option name="isMigrated" value="true" />
|
||||||
<option name="delegatedBuild" value="true" />
|
|
||||||
</component>
|
</component>
|
||||||
<component name="FileEditorManager">
|
<component name="FileEditorManager">
|
||||||
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
|
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
|
||||||
@ -20,16 +19,16 @@
|
|||||||
<entry file="file://$PROJECT_DIR$/pom.xml">
|
<entry file="file://$PROJECT_DIR$/pom.xml">
|
||||||
<provider selected="true" editor-type-id="text-editor">
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
<state relative-caret-position="102">
|
<state relative-caret-position="102">
|
||||||
<caret line="6" column="16" lean-forward="true" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
|
<caret line="6" column="16" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
|
||||||
</state>
|
</state>
|
||||||
</provider>
|
</provider>
|
||||||
</entry>
|
</entry>
|
||||||
</file>
|
</file>
|
||||||
<file pinned="false" current-in-tab="false">
|
<file pinned="false" current-in-tab="true">
|
||||||
<entry file="file://$PROJECT_DIR$/src/main/java/App.java">
|
<entry file="file://$PROJECT_DIR$/src/main/java/App.java">
|
||||||
<provider selected="true" editor-type-id="text-editor">
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
<state relative-caret-position="2111">
|
<state relative-caret-position="156">
|
||||||
<caret line="150" lean-forward="true" selection-start-line="150" selection-end-line="154" selection-end-column="1" />
|
<caret line="16" column="54" lean-forward="true" selection-start-line="16" selection-start-column="54" selection-end-line="16" selection-end-column="54" />
|
||||||
<folding>
|
<folding>
|
||||||
<element signature="imports" expanded="true" />
|
<element signature="imports" expanded="true" />
|
||||||
</folding>
|
</folding>
|
||||||
@ -37,9 +36,37 @@
|
|||||||
</provider>
|
</provider>
|
||||||
</entry>
|
</entry>
|
||||||
</file>
|
</file>
|
||||||
<file pinned="false" current-in-tab="true">
|
<file pinned="false" current-in-tab="false">
|
||||||
<entry file="file://$PROJECT_DIR$/src/main/java/Article.java">
|
<entry file="file://$PROJECT_DIR$/src/main/java/Article.java">
|
||||||
<provider selected="true" editor-type-id="text-editor" />
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
|
<state relative-caret-position="493">
|
||||||
|
<caret line="59" column="28" selection-start-line="59" selection-start-column="18" selection-end-line="59" selection-end-column="28" />
|
||||||
|
<folding>
|
||||||
|
<element signature="e#511#512#0" expanded="true" />
|
||||||
|
<element signature="e#540#541#0" expanded="true" />
|
||||||
|
<element signature="e#584#585#0" expanded="true" />
|
||||||
|
<element signature="e#618#619#0" expanded="true" />
|
||||||
|
<element signature="e#656#657#0" expanded="true" />
|
||||||
|
<element signature="e#684#685#0" expanded="true" />
|
||||||
|
<element signature="e#724#725#0" expanded="true" />
|
||||||
|
<element signature="e#754#755#0" expanded="true" />
|
||||||
|
<element signature="e#800#801#0" expanded="true" />
|
||||||
|
<element signature="e#836#837#0" expanded="true" />
|
||||||
|
<element signature="e#872#873#0" expanded="true" />
|
||||||
|
<element signature="e#910#911#0" expanded="true" />
|
||||||
|
<element signature="e#945#946#0" expanded="true" />
|
||||||
|
<element signature="e#981#982#0" expanded="true" />
|
||||||
|
<element signature="e#1023#1024#0" expanded="true" />
|
||||||
|
<element signature="e#1063#1064#0" expanded="true" />
|
||||||
|
<element signature="e#1100#1101#0" expanded="true" />
|
||||||
|
<element signature="e#1135#1136#0" expanded="true" />
|
||||||
|
<element signature="e#1173#1174#0" expanded="true" />
|
||||||
|
<element signature="e#1209#1210#0" expanded="true" />
|
||||||
|
<element signature="e#1250#1251#0" expanded="true" />
|
||||||
|
<element signature="e#1289#1290#0" expanded="true" />
|
||||||
|
</folding>
|
||||||
|
</state>
|
||||||
|
</provider>
|
||||||
</entry>
|
</entry>
|
||||||
</file>
|
</file>
|
||||||
</leaf>
|
</leaf>
|
||||||
@ -57,9 +84,10 @@
|
|||||||
<component name="IdeDocumentHistory">
|
<component name="IdeDocumentHistory">
|
||||||
<option name="CHANGED_PATHS">
|
<option name="CHANGED_PATHS">
|
||||||
<list>
|
<list>
|
||||||
<option value="$PROJECT_DIR$/src/main/java/Article.java" />
|
|
||||||
<option value="$PROJECT_DIR$/CAu_NLP_2019.iml" />
|
<option value="$PROJECT_DIR$/CAu_NLP_2019.iml" />
|
||||||
<option value="$PROJECT_DIR$/pom.xml" />
|
<option value="$PROJECT_DIR$/pom.xml" />
|
||||||
|
<option value="$PROJECT_DIR$/src/main/java/Article.java" />
|
||||||
|
<option value="$PROJECT_DIR$/src/main/java/tfidf.java" />
|
||||||
<option value="$PROJECT_DIR$/src/main/java/App.java" />
|
<option value="$PROJECT_DIR$/src/main/java/App.java" />
|
||||||
</list>
|
</list>
|
||||||
</option>
|
</option>
|
||||||
@ -76,9 +104,42 @@
|
|||||||
<foldersAlwaysOnTop value="true" />
|
<foldersAlwaysOnTop value="true" />
|
||||||
</navigator>
|
</navigator>
|
||||||
<panes>
|
<panes>
|
||||||
<pane id="Scope" />
|
|
||||||
<pane id="PackagesPane" />
|
<pane id="PackagesPane" />
|
||||||
<pane id="ProjectPane" />
|
<pane id="Scope" />
|
||||||
|
<pane id="ProjectPane">
|
||||||
|
<subPane>
|
||||||
|
<expand>
|
||||||
|
<path>
|
||||||
|
<item name="CAu_NLP_2019" type="b2602c69:ProjectViewProjectNode" />
|
||||||
|
<item name="CAu_NLP_2019" type="462c0819:PsiDirectoryNode" />
|
||||||
|
</path>
|
||||||
|
<path>
|
||||||
|
<item name="CAu_NLP_2019" type="b2602c69:ProjectViewProjectNode" />
|
||||||
|
<item name="CAu_NLP_2019" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="src" type="462c0819:PsiDirectoryNode" />
|
||||||
|
</path>
|
||||||
|
<path>
|
||||||
|
<item name="CAu_NLP_2019" type="b2602c69:ProjectViewProjectNode" />
|
||||||
|
<item name="CAu_NLP_2019" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="src" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="main" type="462c0819:PsiDirectoryNode" />
|
||||||
|
</path>
|
||||||
|
<path>
|
||||||
|
<item name="CAu_NLP_2019" type="b2602c69:ProjectViewProjectNode" />
|
||||||
|
<item name="CAu_NLP_2019" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="src" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="main" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="java" type="462c0819:PsiDirectoryNode" />
|
||||||
|
</path>
|
||||||
|
<path>
|
||||||
|
<item name="CAu_NLP_2019" type="b2602c69:ProjectViewProjectNode" />
|
||||||
|
<item name="CAu_NLP_2019" type="462c0819:PsiDirectoryNode" />
|
||||||
|
<item name="target" type="462c0819:PsiDirectoryNode" />
|
||||||
|
</path>
|
||||||
|
</expand>
|
||||||
|
<select />
|
||||||
|
</subPane>
|
||||||
|
</pane>
|
||||||
</panes>
|
</panes>
|
||||||
</component>
|
</component>
|
||||||
<component name="PropertiesComponent">
|
<component name="PropertiesComponent">
|
||||||
@ -130,8 +191,9 @@
|
|||||||
</component>
|
</component>
|
||||||
<component name="ToolWindowManager">
|
<component name="ToolWindowManager">
|
||||||
<frame x="-8" y="-8" width="1936" height="1096" extended-state="6" />
|
<frame x="-8" y="-8" width="1936" height="1096" extended-state="6" />
|
||||||
|
<editor active="true" />
|
||||||
<layout>
|
<layout>
|
||||||
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.18017058" />
|
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.18176973" />
|
||||||
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
|
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
|
||||||
<window_info id="Image Layers" order="2" />
|
<window_info id="Image Layers" order="2" />
|
||||||
<window_info id="Designer" order="3" />
|
<window_info id="Designer" order="3" />
|
||||||
@ -140,15 +202,15 @@
|
|||||||
<window_info id="Favorites" order="6" side_tool="true" />
|
<window_info id="Favorites" order="6" side_tool="true" />
|
||||||
<window_info anchor="bottom" id="Message" order="0" />
|
<window_info anchor="bottom" id="Message" order="0" />
|
||||||
<window_info anchor="bottom" id="Find" order="1" />
|
<window_info anchor="bottom" id="Find" order="1" />
|
||||||
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.4989339" weight="0.32917964" />
|
<window_info active="true" anchor="bottom" id="Run" order="2" sideWeight="0.49520257" visible="true" weight="0.32917964" />
|
||||||
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
|
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
|
||||||
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
|
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
|
||||||
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
|
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
|
||||||
<window_info anchor="bottom" id="TODO" order="6" />
|
<window_info anchor="bottom" id="TODO" order="6" />
|
||||||
<window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.49946696" weight="0.32917964" />
|
<window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.49946696" weight="0.32917964" />
|
||||||
<window_info anchor="bottom" id="Event Log" order="8" sideWeight="0.5010661" side_tool="true" visible="true" weight="0.32917964" />
|
<window_info anchor="bottom" id="Event Log" order="8" sideWeight="0.50479746" side_tool="true" visible="true" weight="0.32917964" />
|
||||||
<window_info anchor="bottom" id="Version Control" order="9" />
|
<window_info anchor="bottom" id="Version Control" order="9" />
|
||||||
<window_info anchor="bottom" id="Messages" order="10" weight="0.32917964" />
|
<window_info anchor="bottom" id="Messages" order="10" sideWeight="0.4978678" weight="0.32917964" />
|
||||||
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
|
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
|
||||||
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
|
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
|
||||||
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
|
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
|
||||||
@ -188,13 +250,6 @@
|
|||||||
</layout-to-restore>
|
</layout-to-restore>
|
||||||
</component>
|
</component>
|
||||||
<component name="editorHistoryManager">
|
<component name="editorHistoryManager">
|
||||||
<entry file="file://$PROJECT_DIR$/pom.xml">
|
|
||||||
<provider selected="true" editor-type-id="text-editor">
|
|
||||||
<state relative-caret-position="102">
|
|
||||||
<caret line="6" column="16" lean-forward="true" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
|
|
||||||
</state>
|
|
||||||
</provider>
|
|
||||||
</entry>
|
|
||||||
<entry file="file://$PROJECT_DIR$/CAu_NLP_2019.iml">
|
<entry file="file://$PROJECT_DIR$/CAu_NLP_2019.iml">
|
||||||
<provider selected="true" editor-type-id="text-editor">
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
<state relative-caret-position="17">
|
<state relative-caret-position="17">
|
||||||
@ -202,19 +257,61 @@
|
|||||||
</state>
|
</state>
|
||||||
</provider>
|
</provider>
|
||||||
</entry>
|
</entry>
|
||||||
|
<entry file="file://$PROJECT_DIR$/pom.xml">
|
||||||
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
|
<state relative-caret-position="102">
|
||||||
|
<caret line="6" column="16" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
|
||||||
|
</state>
|
||||||
|
</provider>
|
||||||
|
</entry>
|
||||||
|
<entry file="file://$PROJECT_DIR$/src/main/java/Article.java">
|
||||||
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
|
<state relative-caret-position="493">
|
||||||
|
<caret line="59" column="28" selection-start-line="59" selection-start-column="18" selection-end-line="59" selection-end-column="28" />
|
||||||
|
<folding>
|
||||||
|
<element signature="e#511#512#0" expanded="true" />
|
||||||
|
<element signature="e#540#541#0" expanded="true" />
|
||||||
|
<element signature="e#584#585#0" expanded="true" />
|
||||||
|
<element signature="e#618#619#0" expanded="true" />
|
||||||
|
<element signature="e#656#657#0" expanded="true" />
|
||||||
|
<element signature="e#684#685#0" expanded="true" />
|
||||||
|
<element signature="e#724#725#0" expanded="true" />
|
||||||
|
<element signature="e#754#755#0" expanded="true" />
|
||||||
|
<element signature="e#800#801#0" expanded="true" />
|
||||||
|
<element signature="e#836#837#0" expanded="true" />
|
||||||
|
<element signature="e#872#873#0" expanded="true" />
|
||||||
|
<element signature="e#910#911#0" expanded="true" />
|
||||||
|
<element signature="e#945#946#0" expanded="true" />
|
||||||
|
<element signature="e#981#982#0" expanded="true" />
|
||||||
|
<element signature="e#1023#1024#0" expanded="true" />
|
||||||
|
<element signature="e#1063#1064#0" expanded="true" />
|
||||||
|
<element signature="e#1100#1101#0" expanded="true" />
|
||||||
|
<element signature="e#1135#1136#0" expanded="true" />
|
||||||
|
<element signature="e#1173#1174#0" expanded="true" />
|
||||||
|
<element signature="e#1209#1210#0" expanded="true" />
|
||||||
|
<element signature="e#1250#1251#0" expanded="true" />
|
||||||
|
<element signature="e#1289#1290#0" expanded="true" />
|
||||||
|
</folding>
|
||||||
|
</state>
|
||||||
|
</provider>
|
||||||
|
</entry>
|
||||||
|
<entry file="file://$PROJECT_DIR$/src/main/java/TFIDF.java">
|
||||||
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
|
<state relative-caret-position="357">
|
||||||
|
<caret line="21" column="7" lean-forward="true" selection-start-line="21" selection-start-column="7" selection-end-line="21" selection-end-column="7" />
|
||||||
|
</state>
|
||||||
|
</provider>
|
||||||
|
</entry>
|
||||||
<entry file="file://$PROJECT_DIR$/src/main/java/App.java">
|
<entry file="file://$PROJECT_DIR$/src/main/java/App.java">
|
||||||
<provider selected="true" editor-type-id="text-editor">
|
<provider selected="true" editor-type-id="text-editor">
|
||||||
<state relative-caret-position="2111">
|
<state relative-caret-position="156">
|
||||||
<caret line="150" lean-forward="true" selection-start-line="150" selection-end-line="154" selection-end-column="1" />
|
<caret line="16" column="54" lean-forward="true" selection-start-line="16" selection-start-column="54" selection-end-line="16" selection-end-column="54" />
|
||||||
<folding>
|
<folding>
|
||||||
<element signature="imports" expanded="true" />
|
<element signature="imports" expanded="true" />
|
||||||
</folding>
|
</folding>
|
||||||
</state>
|
</state>
|
||||||
</provider>
|
</provider>
|
||||||
</entry>
|
</entry>
|
||||||
<entry file="file://$PROJECT_DIR$/src/main/java/Article.java">
|
|
||||||
<provider selected="true" editor-type-id="text-editor" />
|
|
||||||
</entry>
|
|
||||||
</component>
|
</component>
|
||||||
<component name="masterDetails">
|
<component name="masterDetails">
|
||||||
<states>
|
<states>
|
||||||
|
|||||||
@ -1,9 +1,8 @@
|
|||||||
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.text.DecimalFormat;
|
||||||
import java.util.Scanner;
|
import java.util.*;
|
||||||
import java.util.StringTokenizer;
|
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Document;
|
||||||
@ -13,11 +12,14 @@ import org.jsoup.select.Elements;
|
|||||||
public class App {
|
public class App {
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
String url[] = {"https://www.thesun.co.uk/?s=", "https://www.bbc.co.uk/search?q=", "https://www.skysports.com/search?q="};
|
String url[] = {"https://www.thesun.co.uk/?s=", "https://www.bbc.co.uk/search?q=", "https://www.skysports.com/search?q="};
|
||||||
//ArrayList<String> key = new ArrayList<String>();
|
|
||||||
|
List<List<String>> documents = new ArrayList<>();
|
||||||
|
List<List<String>> result = new ArrayList<>();
|
||||||
|
List<String> result2 = new ArrayList<>();
|
||||||
|
String document;
|
||||||
|
List<String> goodUrl = new ArrayList<>();
|
||||||
Article data = new Article();
|
Article data = new Article();
|
||||||
//key.add("salah");
|
|
||||||
//key.add("Liverpool");
|
|
||||||
//key.add("Champions league");
|
|
||||||
Document doc = null;
|
Document doc = null;
|
||||||
Elements element = null;
|
Elements element = null;
|
||||||
|
|
||||||
@ -28,10 +30,7 @@ public class App {
|
|||||||
for (int j = 0; j < url.length; j++) {
|
for (int j = 0; j < url.length; j++) {
|
||||||
for (int i = 0; i < key.length; i++) {
|
for (int i = 0; i < key.length; i++) {
|
||||||
String urlTmp = url[j] + key[i];
|
String urlTmp = url[j] + key[i];
|
||||||
doc = Jsoup.connect(urlTmp).execute().parse();
|
doc = Jsoup.connect(urlTmp).execute().parse(); // Document에 url 페이지의 데이터를 가져온다.
|
||||||
/* } catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}*/
|
|
||||||
if (url[j].equals("https://www.thesun.co.uk/?s=")) {
|
if (url[j].equals("https://www.thesun.co.uk/?s=")) {
|
||||||
element = doc.select("div.search-results-wrap");
|
element = doc.select("div.search-results-wrap");
|
||||||
for (Element el : element.select(".teaser-item")) {
|
for (Element el : element.select(".teaser-item")) {
|
||||||
@ -42,6 +41,8 @@ public class App {
|
|||||||
String temp = el.select(".search-date").text();
|
String temp = el.select(".search-date").text();
|
||||||
data.setDate(changeDate(temp));
|
data.setDate(changeDate(temp));
|
||||||
data.setSite("The Sun");
|
data.setSite("The Sun");
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -70,12 +71,75 @@ public class App {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
String content = "";
|
||||||
for(int i = 0; i < data.getHowManyData(); i++) {
|
for(int i = 0; i < data.getHowManyData(); i++) {
|
||||||
System.out.println(data.getDate(i));
|
content = "";
|
||||||
System.out.println(data.getHeadline(i));
|
try {
|
||||||
System.out.println(data.getUrl(i));
|
doc = Jsoup.connect(data.getUrl(i)).execute().parse();
|
||||||
System.out.println(data.getSite(i));
|
if (data.getSite(i) == "The Sun") {
|
||||||
|
element = doc.select("div.article__content");
|
||||||
|
for (Element el : element.select("p")) {
|
||||||
|
content += el.text();
|
||||||
|
}
|
||||||
|
data.setContent(content);
|
||||||
|
} else if (data.getSite(i) == "BBC") {
|
||||||
|
element = doc.select("div#story-body");
|
||||||
|
for (Element el : element.select("p")) {
|
||||||
|
content += el.text();
|
||||||
|
}
|
||||||
|
data.setContent(content);
|
||||||
|
} else if (data.getSite(i) == "SKYSPORTS") {
|
||||||
|
element = doc.select("div.article__body");
|
||||||
|
for (Element el : element.select("p")) {
|
||||||
|
if (!el.hasClass("widge-marketing__text")) {
|
||||||
|
content += el.text();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data.setContent(content);
|
||||||
|
}
|
||||||
|
if (data.getContent(i).length() > 0){
|
||||||
|
/*
|
||||||
|
System.out.println(data.getDate(i));
|
||||||
|
System.out.println(data.getHeadline(i));
|
||||||
|
System.out.println(data.getUrl(i));
|
||||||
|
System.out.println(data.getSite(i));
|
||||||
|
*/
|
||||||
|
document = data.getContent((i)).replaceAll("\\s+",",");
|
||||||
|
goodUrl.add(data.getUrl(i));
|
||||||
|
documents.add(new ArrayList<String>(Arrays.asList(document.split(","))));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
System.out.println("No content: " + data.getUrl(i));
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
System.out.println("Something went wrong.: " + e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (int i = 0; i < documents.size(); i++){
|
||||||
|
DecimalFormat df = new DecimalFormat("#.####");
|
||||||
|
TFIDF calculator = new TFIDF();
|
||||||
|
double tfidf = calculator.tfIdf(documents.get(i), documents, key[0]);
|
||||||
|
result.add(Arrays.asList((df.format(tfidf)), goodUrl.get(i)));
|
||||||
|
}
|
||||||
|
for (int i = 0; i < result.size(); i++){
|
||||||
|
System.out.println("TF-IDF: " + result.get(i).get(0) + " Url: " + result.get(i).get(1));
|
||||||
|
}
|
||||||
|
double max;
|
||||||
|
int pos;
|
||||||
|
while (result.size() > 0){
|
||||||
|
pos = 0;
|
||||||
|
max = Double.parseDouble(result.get(0).get(0));
|
||||||
|
for (int i2 = 0; i2 < result.size(); i2++){
|
||||||
|
if (Double.compare(max, Double.parseDouble(result.get(i2).get(0))) < 0){
|
||||||
|
max = Double.parseDouble(result.get(i2).get(0));
|
||||||
|
pos = i2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result2.add(result.get(pos).get(1));
|
||||||
|
result.remove(pos);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < result2.size(); i++){
|
||||||
|
System.out.println("Url: " + result2.get(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -94,61 +158,8 @@ public class App {
|
|||||||
|
|
||||||
public static int changeDate(String date) {
|
public static int changeDate(String date) {
|
||||||
int formdate = 0;
|
int formdate = 0;
|
||||||
String sp[] = date.split(" ");/*
|
String sp[] = date.split(" ");
|
||||||
formdate += Integer.parseInt(sp[2]) * 10000;
|
formdate = 0;
|
||||||
formdate += Integer.parseInt(sp[0]);
|
|
||||||
|
|
||||||
switch (sp[1]) {
|
|
||||||
case "January":
|
|
||||||
case "Jan":
|
|
||||||
formdate += 100;
|
|
||||||
break;
|
|
||||||
case "February":
|
|
||||||
case "Feb":
|
|
||||||
formdate += 200;
|
|
||||||
break;
|
|
||||||
case "March":
|
|
||||||
case "Mar":
|
|
||||||
formdate += 300;
|
|
||||||
break;
|
|
||||||
case "April":
|
|
||||||
case "Apr":
|
|
||||||
formdate += 400;
|
|
||||||
break;
|
|
||||||
case "May":
|
|
||||||
formdate += 500;
|
|
||||||
break;
|
|
||||||
case "June":
|
|
||||||
case "Jun":
|
|
||||||
formdate += 600;
|
|
||||||
break;
|
|
||||||
case "July":
|
|
||||||
case "Jul":
|
|
||||||
formdate += 700;
|
|
||||||
break;
|
|
||||||
case "August":
|
|
||||||
case "Aug":
|
|
||||||
formdate += 800;
|
|
||||||
break;
|
|
||||||
case "September":
|
|
||||||
case "Sep":
|
|
||||||
formdate += 900;
|
|
||||||
break;
|
|
||||||
case "October":
|
|
||||||
case "Oct":
|
|
||||||
formdate += 1000;
|
|
||||||
break;
|
|
||||||
case "November":
|
|
||||||
case "Nov":
|
|
||||||
formdate += 1100;
|
|
||||||
break;
|
|
||||||
case "December":
|
|
||||||
case "Dec":
|
|
||||||
formdate += 1200;
|
|
||||||
break;
|
|
||||||
|
|
||||||
}*/
|
|
||||||
|
|
||||||
return formdate;
|
return formdate;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -7,12 +7,14 @@ public class Article {
|
|||||||
Headline = new ArrayList<String>();
|
Headline = new ArrayList<String>();
|
||||||
Url = new ArrayList<String>();
|
Url = new ArrayList<String>();
|
||||||
Site = new ArrayList<String>();
|
Site = new ArrayList<String>();
|
||||||
|
Content = new ArrayList<String>();
|
||||||
}
|
}
|
||||||
|
|
||||||
private ArrayList<Integer> date;
|
private ArrayList<Integer> date;
|
||||||
private ArrayList<String> Headline;
|
private ArrayList<String> Headline;
|
||||||
private ArrayList<String> Url;
|
private ArrayList<String> Url;
|
||||||
private ArrayList<String> Site;
|
private ArrayList<String> Site;
|
||||||
|
private ArrayList<String> Content;
|
||||||
|
|
||||||
public void setDate(int num) {
|
public void setDate(int num) {
|
||||||
date.add(num);
|
date.add(num);
|
||||||
@ -30,6 +32,11 @@ public class Article {
|
|||||||
Site.add(site);
|
Site.add(site);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setContent(String content) {
|
||||||
|
Content.add(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public int getHowManyData() {
|
public int getHowManyData() {
|
||||||
return Headline.size();
|
return Headline.size();
|
||||||
}
|
}
|
||||||
@ -45,7 +52,12 @@ public class Article {
|
|||||||
public String getUrl(int num) {
|
public String getUrl(int num) {
|
||||||
return Url.get(num);
|
return Url.get(num);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getSite(int num) {
|
public String getSite(int num) {
|
||||||
return Site.get(num);
|
return Site.get(num);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getContent(int num) {
|
||||||
|
return Content.get(num);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
46
CAu_NLP_2019/src/main/java/TFIDF.java
Normal file
46
CAu_NLP_2019/src/main/java/TFIDF.java
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class TFIDF {
|
||||||
|
/**
|
||||||
|
* @param doc list of strings
|
||||||
|
* @param term String represents a term
|
||||||
|
* @return term frequency of term in document
|
||||||
|
*/
|
||||||
|
public double tf(List<String> doc, String term) {
|
||||||
|
double result = 0;
|
||||||
|
for (String word : doc) {
|
||||||
|
if (term.equalsIgnoreCase(word))
|
||||||
|
result++;
|
||||||
|
}
|
||||||
|
return result / doc.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param docs list of list of strings represents the dataset
|
||||||
|
* @param term String represents a term
|
||||||
|
* @return the inverse term frequency of term in documents
|
||||||
|
*/
|
||||||
|
public double idf(List<List<String>> docs, String term) {
|
||||||
|
double n = 0;
|
||||||
|
for (List<String> doc : docs) {
|
||||||
|
for (String word : doc) {
|
||||||
|
if (term.equalsIgnoreCase(word)) {
|
||||||
|
n++;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Math.log(docs.size() / n);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param doc a text document
|
||||||
|
* @param docs all documents
|
||||||
|
* @param term term
|
||||||
|
* @return the TF-IDF of term
|
||||||
|
*/
|
||||||
|
public double tfIdf(List<String> doc, List<List<String>> docs, String term) {
|
||||||
|
return tf(doc, term) * idf(docs, term);
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
BIN
CAu_NLP_2019/target/classes/TFIDF.class
Normal file
BIN
CAu_NLP_2019/target/classes/TFIDF.class
Normal file
Binary file not shown.
Loading…
Reference in New Issue
Block a user