diff options
Diffstat (limited to '')
| -rw-r--r-- | lib/githubexplorer.pm | 12 | ||||
| -rw-r--r-- | lib/githubexplorer/Profile.pm | 22 |
2 files changed, 31 insertions, 3 deletions
diff --git a/lib/githubexplorer.pm b/lib/githubexplorer.pm index 321d6b1..52a4ec5 100644 --- a/lib/githubexplorer.pm +++ b/lib/githubexplorer.pm @@ -8,7 +8,17 @@ use githubexplorer::Gexf; with qw/githubexplorer::Profile githubexplorer::Repositorie/; -has seed => ( isa => 'ArrayRef', is => 'ro', required => 1 ); +has seed => ( isa => 'ArrayRef', is => 'rw', required => 1, lazy =>1, default => +sub { +my $self = shift; +my $profiles = $self->schema->resultset('Profiles')->search({done => {'!=', 1}}, {order_by => + 'login desc'}); +my @seeds; +while (my $p = $profiles->next) { + push @seeds, $p->login; +} +return \@seeds; +}); has api_login => ( isa => 'Str', is => 'ro', required => 1 ); has api_token => ( isa => 'Str', is => 'ro', required => 1 ); has connect_info => ( isa => 'ArrayRef', is => 'ro', required => 1 ); diff --git a/lib/githubexplorer/Profile.pm b/lib/githubexplorer/Profile.pm index 11b16ca..a2a0b2b 100644 --- a/lib/githubexplorer/Profile.pm +++ b/lib/githubexplorer/Profile.pm @@ -2,6 +2,7 @@ package githubexplorer::Profile; use 5.010; use Moose::Role; use Net::GitHub::V2::Users; +use Try::Tiny; has banned_profiles => ( isa => 'ArrayRef', is => 'ro', default => sub { [qw/gitpan/] } ); @@ -11,6 +12,7 @@ has profiles_to_skip => sub fetch_profile { my ( $self, $login, $depth ) = @_; + say "-> start $login ..."; return if grep { $_ =~ /$login/i } @{ $self->banned_profiles }; return if grep { $_ =~ /$login/i } @{ $self->profiles_to_skip }; @@ -25,6 +27,10 @@ sub fetch_profile { if ( !$profile ) { my $followers = $github->followers(); sleep(1); + if (!$followers || ref $followers ne 'ARRAY') { + sleep(60); + return; + } return if scalar @$followers < 2; say "fetch profile for $login ($depth) ..."; sleep(1); @@ -34,6 +40,7 @@ sub fetch_profile { $self->fetch_profile($login, $depth); } $profile = $self->_create_profile( $login, $github->show, $depth ); + return if !$profile; sleep(2); if ( $self->with_repo ) { $self->fetch_repositories( $profile, $github->list ); @@ -43,9 +50,14 @@ sub fetch_profile { if ( !$profile->done ) { my $local_depth = $depth + 1; # my $followers = $github->followpers(); -# sleep(1); + sleep(1); my $following = $github->following(); + if (!$following || ref $following ne 'ARRAY') { + sleep(60); + return; + } + # foreach my $f (@$followers) { # say $to->login . " is followed by " . $from; # $self->_create_relation($f, $profile, $local_depth); @@ -113,14 +125,20 @@ sub _create_profile { $profile->{depth} = $depth; - my $profile_rs; + my $profile_rs; my $err; + try { $self->schema->txn_do( sub { $profile_rs = $self->schema->resultset('Profiles')->create($profile); } ); +}catch{ + warn $_; + $err = 1; +}; +return if $err; say '-> '.$profile_rs->login . "'s profile created"; return $profile_rs; } |