Skip to content

Commit

Permalink
0.0.0.3 refactoring and Sitemap download.
Browse files Browse the repository at this point in the history
  • Loading branch information
cassiodeveloper committed Jun 20, 2016
1 parent 28fc5fa commit f391aec
Show file tree
Hide file tree
Showing 117 changed files with 2,282 additions and 2,187 deletions.
6 changes: 3 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
language: csharp
solution: RobotsExplorer.sln
script:
language: csharp
solution: RobotsExplorer.sln
script:
- xbuild /p:Configuration="NoTest" RobotsExplorer.sln
402 changes: 201 additions & 201 deletions LICENSE

Large diffs are not rendered by default.

48 changes: 24 additions & 24 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
<img src="https://travis-ci.org/cassiodeveloper/robotsexplorer.svg?branch=master" />
<a href="https://codeclimate.com/github/cassiodeveloper/robotsexplorer">
<img src="https://codeclimate.com/github/cassiodeveloper/robotsexplorer/badges/gpa.svg" />
</a>

Autor: @cassiov1ru5

Ferramenta: Robots Explorer

Objetivos:

Ler o arquivo Robots.txt de um domínio.
Listar os diretórios que o administrador não quer que os mecanismos de busca indexem.
Exploitar os diretório em busca de um HTTP Status Code 200, o que afirma que o diretório está "aberto", no caso o server não foi configurador contra Directory Listing.

Linguagem: C#

Versão: 0.0.1

Github: https://github.com/cassiodeveloper/robotsexplorer

Mini cv:

Formado em Análise e Desenvolvimento de Sistemas e MBA em Arquitetura de Software pela FIAP e especilista em Computação Forense pelo Mackenzie, é bém professor na Universidade Anhanguera e com mais de 10 anos de experiência na área de desenvolvimento de software, e boas práticas no desenvolvimento de código seguro, mobile e segurança da informação.
<img src="https://travis-ci.org/cassiodeveloper/robotsexplorer.svg?branch=master" />
<a href="https://codeclimate.com/github/cassiodeveloper/robotsexplorer">
<img src="https://codeclimate.com/github/cassiodeveloper/robotsexplorer/badges/gpa.svg" />
</a>

Autor: @cassiov1ru5

Ferramenta: Robots Explorer

Objetivos:

Ler o arquivo Robots.txt de um domínio.
Listar os diretórios que o administrador não quer que os mecanismos de busca indexem.
Exploitar os diretório em busca de um HTTP Status Code 200, o que afirma que o diretório está "aberto", no caso o server não foi configurador contra Directory Listing.

Linguagem: C#

Versão: 0.0.1

Github: https://github.com/cassiodeveloper/robotsexplorer

Mini cv:

Formado em Análise e Desenvolvimento de Sistemas e MBA em Arquitetura de Software pela FIAP e especilista em Computação Forense pelo Mackenzie, é bém professor na Universidade Anhanguera e com mais de 10 anos de experiência na área de desenvolvimento de software, e boas práticas no desenvolvimento de código seguro, mobile e segurança da informação.
54 changes: 27 additions & 27 deletions RobotsExplorer.ConfigManager/ConfigManager.cs
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
namespace RobotsExplorer.ConfigManager
{
public static class ConfigManager
{
#region Public Constants

public const string robotPath = "/robots.txt";
public const string urlHelpText = "The {URL} to scan.";
public const string proxyHelpText = "The {PROXY} pattern if you are behind one. Ex: http:user:password:domain:port";
public const string fileListHelpText = "A full {FILE PATH} with a list of domains to scan.";
public const string versionHelpText = "The {VERSION} of Robots Explorer.";
public const string userAgentHelpText = "The string name of a {USER AGENT} to make the request, default is null. List of User Agents updated at: \n useragentstring.com/pages/Browserlist/";
public const string requestQuantityHelpText = "The max {QUANTITY} of total requests, default is no limit.";
public const string requestTimeIntervalHelpText = "The {INTERVAL TIME} in miliseconds between each request, default is 0 miliseconds.";
public const string requestTimeoutHelpText = "The {TIMEOUT} in miliseconds for each request, default is 100.000 miliseconds (100 seconds).";
public const string helpText = "Need help?";
public const string artAscii = @"
____ ___ ____ ___ ______ _____ ___ __ __ ____ _ ___ ____ ___ ____
| \ / \ | \ / \ | |/ ___/ / _]| | || \| | / \ | \ / _]| \
| D ) || o )| || ( \_ / [_ | | || o ) | | || D ) / [_ | D )
| /| O || || O ||_| |_|\__ | | _]|_ _|| _/| |___ | O || / | _]| /
| \| || O || | | | / \ | | [_ | || | | || || \ | [_ | \
| . \ || || | | | \ | | || | || | | || || . \| || . \
|__|\_|\___/ |_____| \___/ |__| \___| |_____||__|__||__| |_____| \___/ |__|\_||_____||__|\_|";

#endregion
}
namespace RobotsExplorer.ConfigManager
{
public static class ConfigManager
{
#region Public Constants

public const string robotPath = "/robots.txt";
public const string urlHelpText = "The {URL} to scan.";
public const string proxyHelpText = "The {PROXY} pattern if you are behind one. Ex: http:user:password:domain:port";
public const string fileListHelpText = "A full {FILE PATH} with a list of domains to scan.";
public const string versionHelpText = "The {VERSION} of Robots Explorer.";
public const string userAgentHelpText = "The string name of a {USER AGENT} to make the request, default is null. List of User Agents updated at: \n useragentstring.com/pages/Browserlist/";
public const string requestQuantityHelpText = "The max {QUANTITY} of total requests, default is no limit.";
public const string requestTimeIntervalHelpText = "The {INTERVAL TIME} in miliseconds between each request, default is 0 miliseconds.";
public const string requestTimeoutHelpText = "The {TIMEOUT} in miliseconds for each request, default is 100.000 miliseconds (100 seconds).";
public const string helpText = "Need help?";
public const string artAscii = @"
____ ___ ____ ___ ______ _____ ___ __ __ ____ _ ___ ____ ___ ____
| \ / \ | \ / \ | |/ ___/ / _]| | || \| | / \ | \ / _]| \
| D ) || o )| || ( \_ / [_ | | || o ) | | || D ) / [_ | D )
| /| O || || O ||_| |_|\__ | | _]|_ _|| _/| |___ | O || / | _]| /
| \| || O || | | | / \ | | [_ | || | | || || \ | [_ | \
| . \ || || | | | \ | | || | || | | || || . \| || . \
|__|\_|\___/ |_____| \___/ |__| \___| |_____||__|__||__| |_____| \___/ |__|\_||_____||__|\_|";

#endregion
}
}
72 changes: 36 additions & 36 deletions RobotsExplorer.ConfigManager/Properties/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -1,36 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("RobotsExplorer.ConfigManager")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("RobotsExplorer.ConfigManager")]
[assembly: AssemblyCopyright("Copyright © 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]

// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5bbf2f63-bc81-453c-9ac5-732cd0edc7c3")]

// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("RobotsExplorer.ConfigManager")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("RobotsExplorer.ConfigManager")]
[assembly: AssemblyCopyright("Copyright © 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]

// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5bbf2f63-bc81-453c-9ac5-732cd0edc7c3")]

// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
110 changes: 55 additions & 55 deletions RobotsExplorer.ConfigManager/RobotsExplorer.ConfigManager.csproj
Original file line number Diff line number Diff line change
@@ -1,56 +1,56 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{72ED040C-083D-4D1E-9763-FFD3CDB2D61F}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>RobotsExplorer.ConfigManager</RootNamespace>
<AssemblyName>RobotsExplorer.ConfigManager</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'NoTest|AnyCPU'">
<OutputPath>bin\NoTest\</OutputPath>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ConfigManager.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{72ED040C-083D-4D1E-9763-FFD3CDB2D61F}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>RobotsExplorer.ConfigManager</RootNamespace>
<AssemblyName>RobotsExplorer.ConfigManager</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'NoTest|AnyCPU'">
<OutputPath>bin\NoTest\</OutputPath>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ConfigManager.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\Visual Studio 2013\Projects\labs.cassiobp.com.br\RobotsExplorer\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\Debug\RobotsExplorer.ConfigManager.pdb
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\Debug\RobotsExplorer.ConfigManager.pdb
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\NoTest\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\NoTest\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\NoTest\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\bin\NoTest\RobotsExplorer.ConfigManager.dll
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\NoTest\RobotsExplorer.ConfigManager.csprojResolveAssemblyReference.cache
C:\Users\mtzcpd1053\Documents\src\RobotsExplorer\RobotsExplorer.ConfigManager\obj\NoTest\RobotsExplorer.ConfigManager.dll
Loading

0 comments on commit f391aec

Please sign in to comment.