Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Iván Aguilar - Movie recommendation solution #26

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Iván Aguilar - Movie recommendation solution #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c506bb5
problem solved
Apr 25, 2021
f63c212
Remove ignored files
Apr 25, 2021
2016269
changes
May 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions big-data.iml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_11">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: org.apache.mahout:mahout-core:0.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.mahout:mahout-math:0.9" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:16.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.12" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.12" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.1" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.xstream:xstream:1.4.4" level="project" />
<orderEntry type="library" name="Maven: xmlpull:xmlpull:1.1.3.1" level="project" />
<orderEntry type="library" name="Maven: xpp3:xpp3_min:1.1.4c" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:4.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:4.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.mahout.commons:commons-cli:2.0-mahout" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.solr:solr-commons-csv:3.5.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-core:1.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.8" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-json:1.8" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.3-1" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
<orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.7.1" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.7.1" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-server:1.8" level="project" />
<orderEntry type="library" name="Maven: asm:asm:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.1" level="project" />
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.0.1" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math:2.1" level="project" />
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:1.4.1" level="project" />
<orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.7" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-simple:1.7.30" level="project" />
</component>
</module>
19 changes: 18 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,19 @@
<groupId>nearsoft.academy</groupId>
<artifactId>big-data</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
</plugins>
</build>
<packaging>jar</packaging>

<name>big-data</name>
<url>http://maven.apache.org</url>
Expand All @@ -26,5 +38,10 @@
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.30</version>
</dependency>
</dependencies>
</project>
117 changes: 117 additions & 0 deletions src/test/java/nearsoft/academy/bigdata/recommendation/MovieRecommender.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package nearsoft.academy.bigdata.recommendation;

import org.apache.commons.collections.BidiMap;
import org.apache.commons.collections.bidimap.TreeBidiMap;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
import org.apache.mahout.cf.taste.impl.neighborhood.ThresholdUserNeighborhood;
import org.apache.mahout.cf.taste.impl.recommender.GenericUserBasedRecommender;
import org.apache.mahout.cf.taste.impl.similarity.PearsonCorrelationSimilarity;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.neighborhood.UserNeighborhood;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.apache.mahout.cf.taste.recommender.UserBasedRecommender;
import org.apache.mahout.cf.taste.similarity.UserSimilarity;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.File;
import java.util.ArrayList;
import java.util.List;

public class MovieRecommender {
private String txtPath;
private String csvPath;
private BidiMap userIds;
private BidiMap productsIds;
private int totalReviews;
private int totalProducts;
private int totalUsers;

public MovieRecommender(String txtPath) throws IOException {
this.txtPath = txtPath;
this.csvPath = "data/movies.csv";
this.userIds = new TreeBidiMap();
this.productsIds = new TreeBidiMap();
this.totalReviews = 0;
this.totalProducts = 0;
this.totalUsers = 0;
this.convertFileToCSV();
}

public int getTotalReviews() {
return this.totalReviews;
}

public int getTotalProducts() {
return this.totalProducts;
}

public int getTotalUsers() {
return this.totalUsers;
}

public List<String> getRecommendationsForUser(String user) throws IOException, TasteException {
UserBasedRecommender recommender = createRecommender(this.csvPath);
int userKey = Integer.parseInt(userIds.get(user).toString());

List<RecommendedItem> recommendations = recommender.recommend(userKey, 10);
List<String> recommendedMovies = new ArrayList<>();
String product;
for (RecommendedItem recommendation : recommendations) {
product = this.productsIds.getKey((int) recommendation.getItemID()).toString();
recommendedMovies.add(product);
}

return recommendedMovies;
}

private UserBasedRecommender createRecommender(String path) throws IOException, TasteException {
DataModel model = new FileDataModel(new File(path));
UserSimilarity similarity = new PearsonCorrelationSimilarity(model);
UserNeighborhood neighborhood = new ThresholdUserNeighborhood(0.1, similarity, model);
UserBasedRecommender recommender = new GenericUserBasedRecommender(model, neighborhood, similarity);

return recommender;
}

public void convertFileToCSV() throws IOException {
BufferedReader br = new BufferedReader(new FileReader(this.txtPath));
BufferedWriter bw = new BufferedWriter(new FileWriter(this.csvPath));

String line;
String[] newLine = {"", "", ""};
while((line = br.readLine()) != null) {
String[] values = line.split(": ", -1);
if(values[0].equals("review/userId")) {
if(this.userIds.get(values[1]) == null) {
this.userIds.put(values[1], this.totalUsers);
newLine[0] = this.userIds.get(values[1]).toString();
this.totalUsers++;
} else {
newLine[0] = this.userIds.get(values[1]).toString();
}
}
if(values[0].equals("product/productId")) {
if(this.productsIds.get(values[1]) == null) {
this.productsIds.put(values[1], this.totalProducts);
newLine[1] = this.productsIds.get(values[1]).toString();
this.totalProducts++;
} else {
newLine[1] = this.productsIds.get(values[1]).toString();
}
}
if(values[0].equals("review/score")) {
newLine[2] = values[1];
bw.write(newLine[0] + "," + newLine[1] + "," + newLine[2] + '\n');
this.totalReviews++;
}
}
br.close();
bw.close();
}

}
3 changes: 2 additions & 1 deletion src/test/java/nearsoft/academy/bigdata/recommendation/MovieRecommenderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ public class MovieRecommenderTest {
public void testDataInfo() throws IOException, TasteException {
//download movies.txt.gz from
// http://snap.stanford.edu/data/web-Movies.html
MovieRecommender recommender = new MovieRecommender("/path/to/movies.txt.gz");

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are allowed to change how you read the recommender's data, but you are not allowed to add or make any other changes in the test file. Please remove recommender.convertFileToCSV(); and make sure to make the required changes so that your class still passes all tests.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed, I removed it from the test file and modified the MovieRecommender, it still passed all the tests.

MovieRecommender recommender = new MovieRecommender("data/movies.txt");
assertEquals(7911684, recommender.getTotalReviews());
assertEquals(253059, recommender.getTotalProducts());
assertEquals(889176, recommender.getTotalUsers());
Expand Down
Binary file added target/test-classes/nearsoft/academy/bigdata/recommendation/MovieRecommender.class
View file
Open in desktop
Binary file not shown.
Binary file added target/test-classes/nearsoft/academy/bigdata/recommendation/MovieRecommenderTest.class
View file
Open in desktop
Binary file not shown.