From fdd30a440382e1366f438341b240882704e921f4 Mon Sep 17 00:00:00 2001 From: Jacklyn J Videira Date: Wed, 6 Feb 2019 12:27:51 -0500 Subject: [PATCH] Update Main.java --- MHealth_GroupFBScraper/src/Main.java | 124 ++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 13 deletions(-) diff --git a/MHealth_GroupFBScraper/src/Main.java b/MHealth_GroupFBScraper/src/Main.java index 2a84a62..6271890 100644 --- a/MHealth_GroupFBScraper/src/Main.java +++ b/MHealth_GroupFBScraper/src/Main.java @@ -12,9 +12,12 @@ import java.util.Scanner; import com.opencsv.CSVWriter; import com.restfb.*; +import com.restfb.types.Comment; import com.restfb.types.Group; import com.restfb.types.Likes; +import com.restfb.types.Likes.LikeItem; import com.restfb.types.Post; +import com.restfb.types.Reactions.ReactionItem; /** * TODO: @@ -29,8 +32,12 @@ public class Main { */ public static void main(String[] args) { - String accessToken = "EAALA0FZC8s14BACP7C9wRohgnBbgGbMmmCnZBjPDcQbGf6ZA1oro7vyZB9ABBYyjKEABrOZA8UNT7rhTW9CDzEZANKzxPdD3JKkbCcL6KpiUmke0gXLIxbE23ZCn91hECSF8VH0oWDPtGeA2Hxmcp6erocnSiQMCFYiY5LHPvJQefNPci5m13QHKE00z2GybB4cIu4mnpZBOJwZDZD"; + String accessToken = "EAALA0FZC8s14BAPWiegZAXVZApHZBVD2kRMva0TqWhTvZCKT2AMzda7pCSq7B4wltaO5pUuDL39nspxYaxN1AezZAlFTgDOWh2iakQ1yT0hZAUplJLfc8MvZArq8xiaIz0PKVeJvGonPaeZA0dg4w97ZCT41CtVkrljwFrqXOVQd9Hl4QkUvLhjxJ6HNjFDBItcZCYZD"; + // Token expires daily - facebook auth needed for permanent one or app needs to login with facebook + // Just MY access token for now - should be replaced with an admin + String[] userAccessTokens = new String[] {accessToken}; + Scanner input = new Scanner(System.in); FacebookClient fbClient = new DefaultFacebookClient(accessToken, Version.VERSION_3_1); @@ -47,9 +54,9 @@ public class Main { //if(response.equals("Yes")) { // Removed above lines for testing (will need them in the future but not now since I'm only testing on one group) if(currGroup.getId().equals("250530099176185")){ - Connection postFeed = fbClient.fetchConnection(currGroup.getId()+"/feed", Post.class, Parameter.with("fields","from,actions,message,likes,reactions,story,type,link,picture,created_time")); + Connection postFeed = fbClient.fetchConnection(currGroup.getId()+"/feed", Post.class, Parameter.with("fields","from,actions,message,likes,reactions,story,type,link,picture,created_time,comments")); // Note - ask Jess if they want to be able to specify a start date / end date when pulling data - writeCSVData(currGroup, postFeed); + writeCSVData(currGroup, postFeed, fbClient, userAccessTokens); } } } @@ -57,16 +64,18 @@ public class Main { } - public static void writeCSVData(Group currGroup, Connection postFeed) + public static void writeCSVData(Group currGroup, Connection postFeed, FacebookClient fbClient, String[] userApprovals) { // TODO - Finish // File Name should be of format: Health Chat_Data Type_Group Name_Date Pulled.csv - /* Files needed: + + /* + * + * Files needed: * comments - headers: PostId, Id, UserId, UserName, CreatedTime, Message * chat_feed - headers: Id, UserId, UserName, CreatedTime, Status Type, Message, Story, Link, Picture * likes - headers: ObjectId, UserId, UserName * reactions - headers: ObjectId, UserId, UserName, Type - * shares - headers: Id, UserId, UserName, CreatedTime, StatusType, Message, Story, Link, Picture * */ Iterator> it = postFeed.iterator(); @@ -75,20 +84,51 @@ public class Main { DateFormat csvDateFormat = new SimpleDateFormat("yyyy-mm-dd hh:mm:ss"); try { // Should first create a folder then put all of these files in that folder with the date - File file = new File("Health Chat_chat_feed_"+currGroup.getName()+"_"+dtf.format(now)+".csv"); // Filepath needs to be more specific in final + // Create the chat_feed csv file + File file = new File("chat_feed "+currGroup.getName()+"_"+dtf.format(now)+".csv"); // Filepath needs to be more specific in final FileWriter outputfile = new FileWriter(file); // create CSVWriter object filewriter object as parameter CSVWriter chatWriter = new CSVWriter(outputfile); List chat_data = new ArrayList(); chat_data.add(new String[] {"Id", "UserId", "UserName", "CreatedTime", "StatusType", "Message", "Story", "Link", "Picture"}); + // create the likes csv file + File likeFile = new File("likes_feed "+currGroup.getName()+"_"+dtf.format(now)+".csv"); // Filepath needs to be more specific in final + FileWriter likeOutputfile = new FileWriter(likeFile); + CSVWriter likeWriter = new CSVWriter(likeOutputfile); + List like_data = new ArrayList(); + like_data.add(new String[] {"ObjectId", "UserId", "UserName"}); + + // create the likes csv file + File reactionFile = new File("reactions_feed "+currGroup.getName()+"_"+dtf.format(now)+".csv"); // Filepath needs to be more specific in final + FileWriter reactionOutputfile = new FileWriter(reactionFile); + CSVWriter reactionWriter = new CSVWriter(reactionOutputfile); + List reaction_data = new ArrayList(); + reaction_data.add(new String[] {"ObjectId", "UserId", "UserName", "Type"}); + + + // create the comments csv file + File commentFile = new File("comments_feed "+currGroup.getName()+"_"+dtf.format(now)+".csv"); // Filepath needs to be more specific in final + FileWriter commentOutputFile = new FileWriter(commentFile); + CSVWriter commentWriter = new CSVWriter(commentOutputFile); + List comment_data = new ArrayList(); + comment_data.add(new String[] {"PostId", "Id", "UserId", "UserName", "CreatedTime", "Message"}); + while(it.hasNext()) { List feedPage = it.next(); //Create a list of posts from the post feed Y for(Post currPost : feedPage) { String id = currPost.getId(); - String UserId = currPost.getFrom().getId(); - String UserName = currPost.getFrom().getName(); + String UserId; + String UserName; + // Need permissions to get username + try { + UserId = currPost.getFrom().getId(); + UserName = currPost.getFrom().getName(); + } catch(NullPointerException e) { + UserId = "Unauthorized User"; + UserName = "Unauthorized User"; + } String CreatedTime = csvDateFormat.format(currPost.getCreatedTime()); String type = currPost.getType(); String message = currPost.getMessage(); @@ -99,19 +139,77 @@ public class Main { //IF post isnt shared, add to chat_data file chat_data.add(new String[] {id,UserId,UserName,CreatedTime,type,message,story,link,pic}); //else add to shares file - if(currPost.getLikesCount() > 0) { + if(currPost.getLikes() != null) { + // This only recognizes when the current authorized user token likes a post //add to the likes.csv file + for (LikeItem currLike : currPost.getLikes().getData()) { + String LikeUserId; + String LikeUserName; + try { + LikeUserId = currLike.getId(); + LikeUserName = currLike.getName(); + } catch(NullPointerException e) { + LikeUserId = "UserId Unavailable"; + LikeUserName = "UserName Unavailable"; + } + + + like_data.add(new String[] {id, LikeUserId, LikeUserName}); + } + + } - if (currPost.getReactionsCount() > 0) { - //add to the reactions.csv file + + if (currPost.getReactions() != null) { + // This only recognizes when the current authorized user reacts + for (ReactionItem currReact : currPost.getReactions().getData()) { + String rType = currReact.getType(); + String rUserId; + String rUserName; + try { + rUserId = currReact.getId(); + rUserName = currReact.getName(); + } catch(NullPointerException e) { + rUserId = "UserId Unavailable"; + rUserName = "UserName Unavailable"; + } + + reaction_data.add(new String[] {id, rUserId, rUserName, rType}); + } } - if (currPost.getCommentsCount() > 0) { + + if (currPost.getComments() != null) { //add to the comments.csv file + for (Comment currComment : currPost.getComments().getData()) { + // postid is id + String commentID = currComment.getId(); + // Can't use getFrom with out user permissions + String commentUserId; + String commentUserName; + try { + commentUserId = currComment.getFrom().getId(); + commentUserName = currComment.getFrom().getName(); + } catch(NullPointerException e) { + commentUserId = "UserId Unavailable"; + commentUserName = "UserName Unavailable"; + } + String commentCreatedTime = csvDateFormat.format(currComment.getCreatedTime()); + String commentMessage = currComment.getMessage(); + comment_data.add(new String[] {id,commentID,commentUserId,commentUserName,commentCreatedTime,commentMessage}); + // check if the comment has any likes or reacts + + } } } } chatWriter.writeAll(chat_data); + likeWriter.writeAll(like_data); + commentWriter.writeAll(comment_data); + reactionWriter.writeAll(reaction_data); chatWriter.close(); + likeWriter.close(); + commentWriter.close(); + reactionWriter.close(); } catch (IOException e) { // TODO Auto-generated catch block